public async Task TestDelete()
{
string flowName = "localconfiggentest";
var testingConfig = await File.ReadAllTextAsync(@"Resource\flowSaved.json");
await DesignTimeStorage.SaveByName(flowName, testingConfig, FlowDataManager.DataCollectionName);
// generate runtime configs
var result = await this.RuntimeConfigGeneration.GenerateRuntimeConfigs(flowName);
var runtimeConfigFolderUri = result.Properties?.GetValueOrDefault(PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder, null);
_runtimeConfigFolder = new System.Uri(runtimeConfigFolderUri.ToString()).AbsolutePath;
Assert.IsTrue(result.IsSuccess);
// Invoke delete
var runtimeDeleteResult = await RuntimeConfigGeneration.DeleteConfigs(flowName);
Ensure.IsSuccessResult(runtimeDeleteResult);
// Ensure flow config doesn't exist anymore
var flowConfigs = await this.DesignTimeStorage.GetAll(FlowDataManager.DataCollectionName);
Assert.IsTrue(flowConfigs.Count() == 0);
// Ensure job config doesn't exist anymore
var jobConfigs = await this.DesignTimeStorage.GetAll(SparkJobData.DataCollectionName);
Assert.IsTrue(jobConfigs.Count() == 0);
// Ensure runtime configs folder doesn't exist anymore
Assert.IsTrue(!Directory.Exists(_runtimeConfigFolder));
}
public async Task TestEndToEndGeneration()
{
var flowName = "localconfiggentest";
var testingConfig = await File.ReadAllTextAsync(@"Resource\flowSaved.json");
await DesignTimeStorage.SaveByName(flowName, testingConfig, FlowDataManager.DataCollectionName);
// generate runtime configs
var result = await this.RuntimeConfigGeneration.GenerateRuntimeConfigs(flowName);
var runtimeConfigFolderUri = result.Properties?.GetValueOrDefault(PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder, null);
_runtimeConfigFolder = new System.Uri(runtimeConfigFolderUri.ToString()).AbsolutePath;
Assert.IsTrue(result.IsSuccess);
// verify output schema file is expected
var expectedSchema = JsonConfig.From(await File.ReadAllTextAsync(@"Resource\schema.json"));
var p = ResourcePathUtil.Combine(_runtimeConfigFolder, "inputschema.json");
var pp = new System.Uri(p).AbsolutePath;
var actualSchema = JsonConfig.From(File.ReadAllText(Path.Combine(_runtimeConfigFolder, "inputschema.json")));
foreach (var match in JsonConfig.Match(expectedSchema, actualSchema))
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}");
}
// verify output projection file is expected
var expectedProjection = await File.ReadAllTextAsync(@"Resource\projection.txt");
var actualProjection = File.ReadAllText(Path.Combine(_runtimeConfigFolder, "projection.txt"));
Assert.AreEqual(expected: expectedProjection, actual: actualProjection);
// verify transform file is exepcted
var expectedTransform = await File.ReadAllTextAsync(@"Resource\localconfiggentest-combined.txt");
var actualTransform = File.ReadAllText(Path.Combine(_runtimeConfigFolder, "localconfiggentest-combined.txt"));
Assert.AreEqual(expected: expectedTransform, actual: actualTransform);
// verify output configuration
var actualConf = PropertiesDictionary.From(File.ReadAllText(Path.Combine(_runtimeConfigFolder, $"{flowName}.conf")));
Assert.IsTrue(actualConf.Count == 74, $"Actual entries in .conf not as expected. Expected=65, Got={actualConf.Count}");
// verify metrics
var expectedConfig = JsonConfig.From(await File.ReadAllTextAsync(@"Resource\flowStarted.json"));
var actualConfig = JsonConfig.From(await this.DesignTimeStorage.GetByName(flowName, FlowDataManager.DataCollectionName));
foreach (var match in JsonConfig.Match(expectedConfig, actualConfig))
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}");
}
}
public async Task EndToEndGenerationCustom()
{
var flowName = "customconfiggentest";
var testingConfig = await File.ReadAllTextAsync(@"Resource\customflow.json");
await DesignTimeStorage.SaveByName(flowName, testingConfig, FlowDataManager.DataCollectionName);
await CommonData.Add("defaultFlowConfig", @"Resource\customflow.json");
await CommonData.Add("flattener", @"Resource\customFlattenerConfig.json");
await CommonData.Add("defaultJobTemplate", @"Resource\sparkJobTemplate.json");
var result = await this.RuntimeConfigGeneration.GenerateRuntimeConfigs(flowName);
var runtimeConfigFolder = result.Properties?.GetValueOrDefault(PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder, null);
Assert.IsTrue(result.IsSuccess);
Assert.AreEqual(expected: 2, actual: RuntimeStorage.Cache.Count);
var jobConfigDestinationFolder = runtimeConfigFolder?.ToString().Split("Generation_").First();
// Verify output configuration is expected
var actualConf = PropertiesDictionary.From(this.RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), "customconfiggentest1.conf")]);
var expectedConf = PropertiesDictionary.From(await File.ReadAllTextAsync(@"Resource\customJobConfig1.conf"));
var matches = PropertiesDictionary.Match(expectedConf, actualConf).ToList();
foreach (var match in matches)
{
Console.WriteLine($"prop:{match.Item1 ?? "null"}, expected:<{match.Item2 ?? "null"}>, actual:<{match.Item3 ?? "null"}>");
}
foreach (var match in matches)
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"property:{match.Item1}");
}
//verify second job conf is generated as expected
flowName = "customconfiggentest2";
actualConf = PropertiesDictionary.From(this.RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), "customconfiggentest2.conf")]);
expectedConf = PropertiesDictionary.From(await File.ReadAllTextAsync(@"Resource\customJobConfig2.conf"));
matches = PropertiesDictionary.Match(expectedConf, actualConf).ToList();
foreach (var match in matches)
{
Console.WriteLine($"prop:{match.Item1 ?? "null"}, expected:<{match.Item2 ?? "null"}>, actual:<{match.Item3 ?? "null"}>");
}
foreach (var match in matches)
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"property:{match.Item1}");
}
Cleanup();
}
public async Task TestBatchSchedule()
{
var flowName = "configgenbatchtest";
var testingConfig = await File.ReadAllTextAsync(@"Resource\batchFlow.json");
var current = DateTime.UtcNow;
var startTimeConfig = current.AddDays(-1);
var endTimeConfig = current.AddDays(1);
var normalizedStartTime = startTimeConfig.Add(-startTimeConfig.TimeOfDay);
testingConfig = testingConfig.Replace("${startTime}", startTimeConfig.ToString("o"));
testingConfig = testingConfig.Replace("${endTime}", endTimeConfig.ToString("o"));
await DesignTimeStorage.SaveByName(flowName, testingConfig, FlowDataManager.DataCollectionName);
await CommonData.Add("defaultJobTemplate", @"Resource\batchSparkJobTemplate.json");
await CommonData.Add(ConfigFlattenerManager.DefaultConfigName, @"Resource\flattenerConfig.json");
await CommonData.Add(FlowDataManager.CommonDataName_DefaultFlowConfig, @"Resource\batchFlowDefault.json");
var result = await this.RuntimeConfigGeneration.GenerateRuntimeConfigs(flowName);
var runtimeConfigFolder = result.Properties?.GetValueOrDefault(PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder, null);
Assert.IsTrue(result.IsSuccess);
Assert.AreEqual(expected: 8, actual: RuntimeStorage.Cache.Count);
var processTime = Regex.Replace(normalizedStartTime.ToString("s"), "[^0-9]", "");
var actualConf = PropertiesDictionary.From(this.RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString() + $@"/Recurring/{processTime}", $"{flowName}.conf")]);
var conf = await File.ReadAllTextAsync(@"Resource\batchJobConfig.conf");
conf = conf.Replace("${processTime}", processTime);
conf = conf.Replace("${startTime}", normalizedStartTime.AddDays(-2).ToString("o"));
conf = conf.Replace("${endTime}", normalizedStartTime.AddDays(1).AddSeconds(-1).ToString("o"));
var expectedConf = PropertiesDictionary.From(conf);
var matches = PropertiesDictionary.Match(expectedConf, actualConf).ToList();
foreach (var match in matches)
{
Console.WriteLine($"prop:{match.Item1 ?? "null"}, expected:<{match.Item2 ?? "null"}>, actual:<{match.Item3 ?? "null"}>");
}
foreach (var match in matches)
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"property:{match.Item1}");
}
Cleanup();
}
public async Task DatabricksTestEndToEndGeneration()
{
var flowName = "dbconfiggentest";
var testingConfig = await File.ReadAllTextAsync(@"Resource\databricksFlowSaved.json");
await DesignTimeStorage.SaveByName(flowName, testingConfig, FlowDataManager.DataCollectionName);
await CommonData.Add("defaultJobTemplate", @"Resource\sparkJobTemplate.json");
await CommonData.Add(ConfigFlattenerManager.DefaultConfigName, @"Resource\flattenerConfig.json");
await CommonData.Add(FlowDataManager.CommonDataName_DefaultFlowConfig, @"Resource\flowDefault.json");
var result = await this.RuntimeConfigGeneration.GenerateRuntimeConfigs(flowName);
var runtimeConfigFolder = result.Properties?.GetValueOrDefault(PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder, null);
Assert.IsTrue(result.IsSuccess);
Assert.AreEqual(expected: 4, actual: RuntimeStorage.Cache.Count);
// verify output schema file is expected
var expectedSchema = JsonConfig.From(await File.ReadAllTextAsync(@"Resource\schema.json"));
var actualSchema = JsonConfig.From(RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), "inputschema.json")]);
foreach (var match in JsonConfig.Match(expectedSchema, actualSchema))
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}");
}
// verify output projection file is expected
var expectedProjection = await File.ReadAllTextAsync(@"Resource\projection.txt");
var actualProjection = RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), "projection.txt")];
Assert.AreEqual(expected: expectedProjection, actual: actualProjection);
// verify transform file is exepcted
var expectedTransform = await File.ReadAllTextAsync(@"Resource\configgentest-combined.txt");
var actualTransform = RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), "dbconfiggentest-combined.txt")];
Assert.AreEqual(expected: expectedTransform, actual: actualTransform);
// Verify output configuration is expected
var actualConf = PropertiesDictionary.From(this.RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), $"{flowName}.conf")]);
var expectedConf = PropertiesDictionary.From(await File.ReadAllTextAsync(@"Resource\databricksJobConfig.conf"));
var matches = PropertiesDictionary.Match(expectedConf, actualConf).ToList();
foreach (var match in matches)
{
Console.WriteLine($"prop:{match.Item1 ?? "null"}, expected:<{match.Item2 ?? "null"}>, actual:<{match.Item3 ?? "null"}>");
}
foreach (var match in matches)
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"property:{match.Item1}");
}
// Verify metrics
var expectedConfig = JsonConfig.From(await File.ReadAllTextAsync(@"Resource\databricksFlowStarted.json"));
var actualConfig = JsonConfig.From(await this.DesignTimeStorage.GetByName(flowName, FlowDataManager.DataCollectionName));
foreach (var match in JsonConfig.Match(expectedConfig, actualConfig))
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}");
}
Cleanup();
}
