public async Task TestFlatten() { var config = await File.ReadAllTextAsync(@"Resource\Flattener\config.json"); var flattener = ConfigFlattener.From(JsonConfig.From(config)); var input = await File.ReadAllTextAsync(@"Resource\Flattener\input.json"); var inputJson = JsonConfig.From(input); var output = flattener.Flatten(inputJson); var actualConf = PropertiesDictionary.From(output); var expected = PropertiesDictionary.From(await File.ReadAllTextAsync(@"Resource\Flattener\output.conf")); var matches = PropertiesDictionary.Match(expected, actualConf).ToList(); foreach (var match in matches) { Console.WriteLine($"property:'{match.Item1}', expected:'{match.Item2}', actual:'{match.Item3}'"); } foreach (var match in matches) { Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"property:'{match.Item1}'"); } }
public async Task TestEndToEndGeneration() { var flowName = "localconfiggentest"; var testingConfig = await File.ReadAllTextAsync(@"Resource\flowSaved.json"); await DesignTimeStorage.SaveByName(flowName, testingConfig, FlowDataManager.DataCollectionName); // generate runtime configs var result = await this.RuntimeConfigGeneration.GenerateRuntimeConfigs(flowName); var runtimeConfigFolderUri = result.Properties?.GetValueOrDefault(PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder, null); _runtimeConfigFolder = new System.Uri(runtimeConfigFolderUri.ToString()).AbsolutePath; Assert.IsTrue(result.IsSuccess); // verify output schema file is expected var expectedSchema = JsonConfig.From(await File.ReadAllTextAsync(@"Resource\schema.json")); var p = ResourcePathUtil.Combine(_runtimeConfigFolder, "inputschema.json"); var pp = new System.Uri(p).AbsolutePath; var actualSchema = JsonConfig.From(File.ReadAllText(Path.Combine(_runtimeConfigFolder, "inputschema.json"))); foreach (var match in JsonConfig.Match(expectedSchema, actualSchema)) { Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}"); } // verify output projection file is expected var expectedProjection = await File.ReadAllTextAsync(@"Resource\projection.txt"); var actualProjection = File.ReadAllText(Path.Combine(_runtimeConfigFolder, "projection.txt")); Assert.AreEqual(expected: expectedProjection, actual: actualProjection); // verify transform file is exepcted var expectedTransform = await File.ReadAllTextAsync(@"Resource\localconfiggentest-combined.txt"); var actualTransform = File.ReadAllText(Path.Combine(_runtimeConfigFolder, "localconfiggentest-combined.txt")); Assert.AreEqual(expected: expectedTransform, actual: actualTransform); // verify output configuration var actualConf = PropertiesDictionary.From(File.ReadAllText(Path.Combine(_runtimeConfigFolder, $"{flowName}.conf"))); Assert.IsTrue(actualConf.Count == 74, $"Actual entries in .conf not as expected. Expected=65, Got={actualConf.Count}"); // verify metrics var expectedConfig = JsonConfig.From(await File.ReadAllTextAsync(@"Resource\flowStarted.json")); var actualConfig = JsonConfig.From(await this.DesignTimeStorage.GetByName(flowName, FlowDataManager.DataCollectionName)); foreach (var match in JsonConfig.Match(expectedConfig, actualConfig)) { Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}"); } }
public async Task EndToEndGenerationCustom() { var flowName = "customconfiggentest"; var testingConfig = await File.ReadAllTextAsync(@"Resource\customflow.json"); await DesignTimeStorage.SaveByName(flowName, testingConfig, FlowDataManager.DataCollectionName); await CommonData.Add("defaultFlowConfig", @"Resource\customflow.json"); await CommonData.Add("flattener", @"Resource\customFlattenerConfig.json"); await CommonData.Add("defaultJobTemplate", @"Resource\sparkJobTemplate.json"); var result = await this.RuntimeConfigGeneration.GenerateRuntimeConfigs(flowName); var runtimeConfigFolder = result.Properties?.GetValueOrDefault(PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder, null); Assert.IsTrue(result.IsSuccess); Assert.AreEqual(expected: 2, actual: RuntimeStorage.Cache.Count); var jobConfigDestinationFolder = runtimeConfigFolder?.ToString().Split("Generation_").First(); // Verify output configuration is expected var actualConf = PropertiesDictionary.From(this.RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), "customconfiggentest1.conf")]); var expectedConf = PropertiesDictionary.From(await File.ReadAllTextAsync(@"Resource\customJobConfig1.conf")); var matches = PropertiesDictionary.Match(expectedConf, actualConf).ToList(); foreach (var match in matches) { Console.WriteLine($"prop:{match.Item1 ?? "null"}, expected:<{match.Item2 ?? "null"}>, actual:<{match.Item3 ?? "null"}>"); } foreach (var match in matches) { Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"property:{match.Item1}"); } //verify second job conf is generated as expected flowName = "customconfiggentest2"; actualConf = PropertiesDictionary.From(this.RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), "customconfiggentest2.conf")]); expectedConf = PropertiesDictionary.From(await File.ReadAllTextAsync(@"Resource\customJobConfig2.conf")); matches = PropertiesDictionary.Match(expectedConf, actualConf).ToList(); foreach (var match in matches) { Console.WriteLine($"prop:{match.Item1 ?? "null"}, expected:<{match.Item2 ?? "null"}>, actual:<{match.Item3 ?? "null"}>"); } foreach (var match in matches) { Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"property:{match.Item1}"); } Cleanup(); }
public async Task TestBatchSchedule() { var flowName = "configgenbatchtest"; var testingConfig = await File.ReadAllTextAsync(@"Resource\batchFlow.json"); var current = DateTime.UtcNow; var startTimeConfig = current.AddDays(-1); var endTimeConfig = current.AddDays(1); var normalizedStartTime = startTimeConfig.Add(-startTimeConfig.TimeOfDay); testingConfig = testingConfig.Replace("${startTime}", startTimeConfig.ToString("o")); testingConfig = testingConfig.Replace("${endTime}", endTimeConfig.ToString("o")); await DesignTimeStorage.SaveByName(flowName, testingConfig, FlowDataManager.DataCollectionName); await CommonData.Add("defaultJobTemplate", @"Resource\batchSparkJobTemplate.json"); await CommonData.Add(ConfigFlattenerManager.DefaultConfigName, @"Resource\flattenerConfig.json"); await CommonData.Add(FlowDataManager.CommonDataName_DefaultFlowConfig, @"Resource\batchFlowDefault.json"); var result = await this.RuntimeConfigGeneration.GenerateRuntimeConfigs(flowName); var runtimeConfigFolder = result.Properties?.GetValueOrDefault(PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder, null); Assert.IsTrue(result.IsSuccess); Assert.AreEqual(expected: 8, actual: RuntimeStorage.Cache.Count); var processTime = Regex.Replace(normalizedStartTime.ToString("s"), "[^0-9]", ""); var actualConf = PropertiesDictionary.From(this.RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString() + $@"/Recurring/{processTime}", $"{flowName}.conf")]); var conf = await File.ReadAllTextAsync(@"Resource\batchJobConfig.conf"); conf = conf.Replace("${processTime}", processTime); conf = conf.Replace("${startTime}", normalizedStartTime.AddDays(-2).ToString("o")); conf = conf.Replace("${endTime}", normalizedStartTime.AddDays(1).AddSeconds(-1).ToString("o")); var expectedConf = PropertiesDictionary.From(conf); var matches = PropertiesDictionary.Match(expectedConf, actualConf).ToList(); foreach (var match in matches) { Console.WriteLine($"prop:{match.Item1 ?? "null"}, expected:<{match.Item2 ?? "null"}>, actual:<{match.Item3 ?? "null"}>"); } foreach (var match in matches) { Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"property:{match.Item1}"); } Cleanup(); }
public string Flatten(JsonConfig config) { var dict = PropertiesDictionary.From(_flattenConfig?.FlattenJsonConfig(config)); return(dict?.ToString()); }
public async Task DatabricksTestEndToEndGeneration() { var flowName = "dbconfiggentest"; var testingConfig = await File.ReadAllTextAsync(@"Resource\databricksFlowSaved.json"); await DesignTimeStorage.SaveByName(flowName, testingConfig, FlowDataManager.DataCollectionName); await CommonData.Add("defaultJobTemplate", @"Resource\sparkJobTemplate.json"); await CommonData.Add(ConfigFlattenerManager.DefaultConfigName, @"Resource\flattenerConfig.json"); await CommonData.Add(FlowDataManager.CommonDataName_DefaultFlowConfig, @"Resource\flowDefault.json"); var result = await this.RuntimeConfigGeneration.GenerateRuntimeConfigs(flowName); var runtimeConfigFolder = result.Properties?.GetValueOrDefault(PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder, null); Assert.IsTrue(result.IsSuccess); Assert.AreEqual(expected: 4, actual: RuntimeStorage.Cache.Count); // verify output schema file is expected var expectedSchema = JsonConfig.From(await File.ReadAllTextAsync(@"Resource\schema.json")); var actualSchema = JsonConfig.From(RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), "inputschema.json")]); foreach (var match in JsonConfig.Match(expectedSchema, actualSchema)) { Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}"); } // verify output projection file is expected var expectedProjection = await File.ReadAllTextAsync(@"Resource\projection.txt"); var actualProjection = RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), "projection.txt")]; Assert.AreEqual(expected: expectedProjection, actual: actualProjection); // verify transform file is exepcted var expectedTransform = await File.ReadAllTextAsync(@"Resource\configgentest-combined.txt"); var actualTransform = RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), "dbconfiggentest-combined.txt")]; Assert.AreEqual(expected: expectedTransform, actual: actualTransform); // Verify output configuration is expected var actualConf = PropertiesDictionary.From(this.RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), $"{flowName}.conf")]); var expectedConf = PropertiesDictionary.From(await File.ReadAllTextAsync(@"Resource\databricksJobConfig.conf")); var matches = PropertiesDictionary.Match(expectedConf, actualConf).ToList(); foreach (var match in matches) { Console.WriteLine($"prop:{match.Item1 ?? "null"}, expected:<{match.Item2 ?? "null"}>, actual:<{match.Item3 ?? "null"}>"); } foreach (var match in matches) { Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"property:{match.Item1}"); } // Verify metrics var expectedConfig = JsonConfig.From(await File.ReadAllTextAsync(@"Resource\databricksFlowStarted.json")); var actualConfig = JsonConfig.From(await this.DesignTimeStorage.GetByName(flowName, FlowDataManager.DataCollectionName)); foreach (var match in JsonConfig.Match(expectedConfig, actualConfig)) { Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}"); } Cleanup(); }