public async Task TestEndToEnd()
{
string flowName = "localconfiggentest";
// Point these two settings to your filesystem locations
InitialConfiguration.Set(LocalSparkClient.ConfigSettingName_SparkHomeFolder, @"<LOCAL SPARK HOME FOLDER>");
InitialConfiguration.Set(Constants.ConfigSettingName_LocalRoot, @"<LOCAL ROOT>");
var input = await File.ReadAllTextAsync(@"Resource\guiInputLocal.json");
var inputJson = JsonConfig.From(input);
var result = await this.FlowOperation.SaveFlowConfig(FlowGuiConfig.From(inputJson));
Assert.IsTrue(result.IsSuccess, result.Message);
var result2 = await this.RuntimeConfigGeneration.GenerateRuntimeConfigs(flowName);
var runtimeConfigFolderUri = result2.Properties?.GetValueOrDefault(PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder, null);
_runtimeConfigFolder = new System.Uri(runtimeConfigFolderUri.ToString()).AbsolutePath;
var jobResult1 = await JobOperation.StartJob(flowName);
Assert.IsTrue(jobResult1.IsSuccess);
// Wait for few seconds for the job to do some work
System.Threading.Thread.Sleep(5000);
var jobResult2 = await JobOperation.StopJob(flowName);
Assert.IsTrue(jobResult2.IsSuccess);
}
public async Task TestFlatten()
{
var config = await File.ReadAllTextAsync(@"Resource\Flattener\config.json");
var flattener = ConfigFlattener.From(JsonConfig.From(config));
var input = await File.ReadAllTextAsync(@"Resource\Flattener\input.json");
var inputJson = JsonConfig.From(input);
var output = flattener.Flatten(inputJson);
var actualConf = PropertiesDictionary.From(output);
var expected = PropertiesDictionary.From(await File.ReadAllTextAsync(@"Resource\Flattener\output.conf"));
var matches = PropertiesDictionary.Match(expected, actualConf).ToList();
foreach (var match in matches)
{
Console.WriteLine($"property:'{match.Item1}', expected:'{match.Item2}', actual:'{match.Item3}'");
}
foreach (var match in matches)
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"property:'{match.Item1}'");
}
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
// get a flattener
var flattener = await this.ConfigFlatteners.GetDefault();
if (flattener == null)
{
return("no flattern config, skipped");
}
// flatten each job config
var jobs = flowToDeploy.GetJobs();
if (jobs == null)
{
return("no jobs, skipped");
}
foreach (var job in jobs)
{
foreach (var jc in job.JobConfigs)
{
var jsonContent = job.Tokens.Resolve(jc.Content);
if (jsonContent != null)
{
var json = JsonConfig.From(jsonContent);
jc.Content = flattener.Flatten(json);
}
}
}
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var flowConfig = flowToDeploy.Config;
if (flowConfig.GetGuiConfig()?.Input?.Mode != Constants.InputMode_Batching)
{
return("done");
}
// set the default job config
var defaultJobConfig = JsonConfig.From(flowConfig.CommonProcessor?.Template);
Ensure.NotNull(defaultJobConfig, "defaultJobConfig");
flowToDeploy.SetAttachment(TokenName__DefaultJobConfig, defaultJobConfig);
// Deploy job configs
var jobsToDeploy = flowToDeploy?.GetJobs();
if (jobsToDeploy != null)
{
foreach (var job in jobsToDeploy)
{
await GenerateJobConfigContent(job, job.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder), defaultJobConfig).ConfigureAwait(false);
}
return("done");
}
else
{
await Task.Yield();
return("no jobs, skipped");
}
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var flowConfig = flowToDeploy.Config;
// set the default job config
var defaultJobConfig = JsonConfig.From(flowConfig.CommonProcessor?.Template);
Ensure.NotNull(defaultJobConfig, "defaultJobConfig");
flowToDeploy.SetAttachment(ParameterObjectName_DefaultJobConfig, defaultJobConfig);
// Deploy job configs
var jobsToDeploy = flowToDeploy?.GetJobs();
if (jobsToDeploy != null)
{
foreach (var job in jobsToDeploy)
{
GenerateJobConfigContent(job, job.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder), defaultJobConfig);
}
return("done");
}
else
{
await Task.Yield();
return("no jobs, skipped");
}
}
public async Task <ApiResult> SaveFlow([FromBody] JObject config)
{
try
{
RolesCheck.EnsureWriter(Request, _isLocal);
Ensure.NotNull(config, "config");
var inputJson = JsonConfig.From(config.ToString());
var result = await _flowOperation.SaveFlowConfig(FlowGuiConfig.From(inputJson));
Ensure.IsSuccessResult(result);
if (result.Properties != null && result.Properties.TryGetValue(FlowOperation.FlowConfigPropertyName, out object flowConfig))
{
return(ApiResult.CreateSuccess(JToken.FromObject(flowConfig)));
}
else
{
return(ApiResult.CreateSuccess(JToken.FromObject(string.Empty)));
}
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
return(ApiResult.CreateError(e.Message));
}
}
/// <summary>
/// Replace the tokenized placeholders in the serialized json with the tokens' value from this dictionary,
/// then deserialize and return another instance of <see cref="JsonConfig"/>
/// </summary>
/// <param name="json">input json object to perform the token replacement</param>
/// <returns></returns>
public JsonConfig Resolve(JsonConfig json)
{
if (json == null)
{
return(json);
}
return(JsonConfig.From(Resolve(json.ToString())));
}
public async Task TestEndToEndGeneration()
{
var flowName = "localconfiggentest";
var testingConfig = await File.ReadAllTextAsync(@"Resource\flowSaved.json");
await DesignTimeStorage.SaveByName(flowName, testingConfig, FlowDataManager.DataCollectionName);
// generate runtime configs
var result = await this.RuntimeConfigGeneration.GenerateRuntimeConfigs(flowName);
var runtimeConfigFolderUri = result.Properties?.GetValueOrDefault(PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder, null);
_runtimeConfigFolder = new System.Uri(runtimeConfigFolderUri.ToString()).AbsolutePath;
Assert.IsTrue(result.IsSuccess);
// verify output schema file is expected
var expectedSchema = JsonConfig.From(await File.ReadAllTextAsync(@"Resource\schema.json"));
var p = ResourcePathUtil.Combine(_runtimeConfigFolder, "inputschema.json");
var pp = new System.Uri(p).AbsolutePath;
var actualSchema = JsonConfig.From(File.ReadAllText(Path.Combine(_runtimeConfigFolder, "inputschema.json")));
foreach (var match in JsonConfig.Match(expectedSchema, actualSchema))
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}");
}
// verify output projection file is expected
var expectedProjection = await File.ReadAllTextAsync(@"Resource\projection.txt");
var actualProjection = File.ReadAllText(Path.Combine(_runtimeConfigFolder, "projection.txt"));
Assert.AreEqual(expected: expectedProjection, actual: actualProjection);
// verify transform file is exepcted
var expectedTransform = await File.ReadAllTextAsync(@"Resource\localconfiggentest-combined.txt");
var actualTransform = File.ReadAllText(Path.Combine(_runtimeConfigFolder, "localconfiggentest-combined.txt"));
Assert.AreEqual(expected: expectedTransform, actual: actualTransform);
// verify output configuration
var actualConf = PropertiesDictionary.From(File.ReadAllText(Path.Combine(_runtimeConfigFolder, $"{flowName}.conf")));
Assert.IsTrue(actualConf.Count == 74, $"Actual entries in .conf not as expected. Expected=65, Got={actualConf.Count}");
// verify metrics
var expectedConfig = JsonConfig.From(await File.ReadAllTextAsync(@"Resource\flowStarted.json"));
var actualConfig = JsonConfig.From(await this.DesignTimeStorage.GetByName(flowName, FlowDataManager.DataCollectionName));
foreach (var match in JsonConfig.Match(expectedConfig, actualConfig))
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}");
}
}
public void TestDeserializedFlowGuiConfig()
{
var jsonString = File.ReadAllText(@"Resource\guiInput.json");
var json = JsonConfig.From(jsonString);
var instance = FlowGuiConfig.From(json);
Assert.AreEqual(expected: "configgentest", actual: instance.Name);
Assert.AreEqual(expected: "configgentest", actual: instance.DisplayName);
Assert.AreEqual(expected: "iothub", actual: instance.Input.InputType);
Assert.AreEqual(expected: "8000", actual: instance.Process.JobConfig.JobExecutorMemory);
}
public async Task <Result> UpdatePartialByName(string partial, string field, string name, string collectionName)
{
var key = this.GetKeyName(name, collectionName);
var baseConfig = this._cache[key];
JsonConfig fieldData = JsonConfig.From(partial);
var builder = JsonConfig.StartBuild().WithBase(JsonConfig.From(baseConfig));
var finalBuilder = builder.ReplaceFieldWithConfig(field, fieldData);
var mergedConfig = finalBuilder.Build();
this._cache[key] = mergedConfig.ToString();
return(await this.GetSuccessResult());
}
public async Task <JsonConfig> GetByName(string name)
{
var data = await Storage.GetByName(name, DataCollectionName);
if (data == null)
{
return(null);
}
var content = JsonConfig.From(data).ConvertTo <CommonDataConfig>()?.Content?.ToString();
return(JsonConfig.From(content));
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var expectedConfigContent = await File.ReadAllTextAsync(@"Resource\jobConfig.json");
var expectedJson = JsonConfig.From(expectedConfigContent);
var actualContentContent = flowToDeploy.GetJobs().First().GetTokenString(GenerateJobConfig.TokenName_JobConfigContent);
var actualJson = JsonConfig.From(actualContentContent);
foreach (var match in JsonConfig.Match(expectedJson, actualJson))
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}");
}
return("done");
}
public static JsonConfig ReplaceTokens(JsonConfig config, IDictionary <string, Token> tokens)
{
if (tokens == null || tokens.Count == 0 || config == null)
{
return(config);
}
var text = config.ToString();
foreach (var token in ResolveTokens(tokens))
{
text = ReplaceStringWithToken(text, token.Key, token.Value);
}
return(JsonConfig.From(text));
}
private async Task <Result> InsertTemplateToCommonData(string name, string content)
{
var existing = await CommonData.GetByName(name);
if (existing == null)
{
var result = await CommonData.SaveByName(name, JsonConfig.From(content));
return(result);
}
else
{
//skip if it already exists
return(new SuccessResult($"Already exists, skipped"));
}
}
public async Task <JsonConfig> GetByName(string name)
{
if (!Configuration.TryGet(ComposeConfigSettingName(name), out var filePath))
{
return(null);
}
if (File.Exists(filePath))
{
var content = await File.ReadAllTextAsync(filePath);
return(JsonConfig.From(content));
}
else
{
throw new ConfigGenerationException($"File '{filePath}' does not exist for common data '{name}'");
}
}
public async Task TestConfigSaved()
{
string flowName = "localconfiggentest";
var input = await File.ReadAllTextAsync(@"Resource\guiInput.json");
var inputJson = JsonConfig.From(input);
var result = await this.FlowOperation.SaveFlowConfig(FlowGuiConfig.From(inputJson));
Assert.IsTrue(result.IsSuccess, result.Message);
var actualJson = await this.FlowOperation.GetFlowByName(flowName);
var expectedJson = FlowConfig.From(await File.ReadAllTextAsync(@"Resource\flowSaved.json"));
foreach (var match in EntityConfig.Match(expectedJson, actualJson))
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}");
}
}
public async Task TestConfigSaved()
{
await CommonData.Add(FlowDataManager.CommonDataName_DefaultFlowConfig, @"Resource\flowDefault.json");
var input = await File.ReadAllTextAsync(@"Resource\guiInput.json");
var inputJson = JsonConfig.From(input);
var result = await this.FlowOperation.SaveFlowConfig(FlowGuiConfig.From(inputJson));
Assert.IsTrue(result.IsSuccess, result.Message);
var actualJson = await this.FlowOperation.GetFlowByName("configgentest");
var expectedJson = FlowConfig.From(await File.ReadAllTextAsync(@"Resource\flowSaved.json"));
foreach (var match in EntityConfig.Match(expectedJson, actualJson))
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}");
}
Cleanup();
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var flowConfig = flowToDeploy.Config;
// get a flattener
var flattener = await this.ConfigFlatteners.GetDefault();
if (flattener == null)
{
return("no flattern config, skipped");
}
// flatten each job config
var jobs = flowToDeploy.GetJobs();
if (jobs == null)
{
return("no jobs, skipped");
}
foreach (var job in jobs)
{
var jsonContent = job.GetTokenString(GenerateJobConfig.TokenName_JobConfigContent);
var destFolder = job.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
if (jsonContent != null)
{
var json = JsonConfig.From(jsonContent);
job.SetStringToken(GenerateJobConfig.TokenName_JobConfigContent, flattener.Flatten(json));
var destinationPath = ResourcePathUtil.Combine(destFolder, job.Name + ".conf");
job.SetStringToken(GenerateJobConfig.TokenName_JobConfigFilePath, destinationPath);
}
}
return("done");
}
public static async Task Add(this ICommonDataManager commonData, string name, string filePath)
{
await commonData.SaveByName(name, JsonConfig.From(await File.ReadAllTextAsync(filePath)));
}
public async Task <SparkClusterConfig> GetByNameAsync(string clusterName)
{
return(SparkClusterConfig.From(JsonConfig.From(await Storage.GetByName(clusterName, DataCollectionName))));
}
public async Task DatabricksTestEndToEndGeneration()
{
var flowName = "dbconfiggentest";
var testingConfig = await File.ReadAllTextAsync(@"Resource\databricksFlowSaved.json");
await DesignTimeStorage.SaveByName(flowName, testingConfig, FlowDataManager.DataCollectionName);
await CommonData.Add("defaultJobTemplate", @"Resource\sparkJobTemplate.json");
await CommonData.Add(ConfigFlattenerManager.DefaultConfigName, @"Resource\flattenerConfig.json");
await CommonData.Add(FlowDataManager.CommonDataName_DefaultFlowConfig, @"Resource\flowDefault.json");
var result = await this.RuntimeConfigGeneration.GenerateRuntimeConfigs(flowName);
var runtimeConfigFolder = result.Properties?.GetValueOrDefault(PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder, null);
Assert.IsTrue(result.IsSuccess);
Assert.AreEqual(expected: 4, actual: RuntimeStorage.Cache.Count);
// verify output schema file is expected
var expectedSchema = JsonConfig.From(await File.ReadAllTextAsync(@"Resource\schema.json"));
var actualSchema = JsonConfig.From(RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), "inputschema.json")]);
foreach (var match in JsonConfig.Match(expectedSchema, actualSchema))
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}");
}
// verify output projection file is expected
var expectedProjection = await File.ReadAllTextAsync(@"Resource\projection.txt");
var actualProjection = RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), "projection.txt")];
Assert.AreEqual(expected: expectedProjection, actual: actualProjection);
// verify transform file is exepcted
var expectedTransform = await File.ReadAllTextAsync(@"Resource\configgentest-combined.txt");
var actualTransform = RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), "dbconfiggentest-combined.txt")];
Assert.AreEqual(expected: expectedTransform, actual: actualTransform);
// Verify output configuration is expected
var actualConf = PropertiesDictionary.From(this.RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), $"{flowName}.conf")]);
var expectedConf = PropertiesDictionary.From(await File.ReadAllTextAsync(@"Resource\databricksJobConfig.conf"));
var matches = PropertiesDictionary.Match(expectedConf, actualConf).ToList();
foreach (var match in matches)
{
Console.WriteLine($"prop:{match.Item1 ?? "null"}, expected:<{match.Item2 ?? "null"}>, actual:<{match.Item3 ?? "null"}>");
}
foreach (var match in matches)
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"property:{match.Item1}");
}
// Verify metrics
var expectedConfig = JsonConfig.From(await File.ReadAllTextAsync(@"Resource\databricksFlowStarted.json"));
var actualConfig = JsonConfig.From(await this.DesignTimeStorage.GetByName(flowName, FlowDataManager.DataCollectionName));
foreach (var match in JsonConfig.Match(expectedConfig, actualConfig))
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}");
}
Cleanup();
}
