public void Move()
{
while (!stop)
{
if (d != null)
{
Thread.Sleep(30);
if (Num == 0)
{
if (X < width / 2)
{
X += dx;
Y += dy;
}
else
{
d.Add(Num, this);
this.Stop();
}
}
if (Num == 1)
{
if (X > width / 2)
{
X += dx;
Y += dy;
}
else
{
d.Add(Num, this);
this.Stop();
}
}
if (Num == 2)
{
if (Y < heigth / 2)
{
X += dx;
Y += dy;
}
else
{
d.Add(Num, this);
this.Stop();
}
}
}
}
}
public async Task EndToEndGenerationCustom()
{
var flowName = "customconfiggentest";
var testingConfig = await File.ReadAllTextAsync(@"Resource\customflow.json");
await DesignTimeStorage.SaveByName(flowName, testingConfig, FlowDataManager.DataCollectionName);
await CommonData.Add("defaultFlowConfig", @"Resource\customflow.json");
await CommonData.Add("flattener", @"Resource\customFlattenerConfig.json");
await CommonData.Add("defaultJobTemplate", @"Resource\sparkJobTemplate.json");
var result = await this.RuntimeConfigGeneration.GenerateRuntimeConfigs(flowName);
var runtimeConfigFolder = result.Properties?.GetValueOrDefault(PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder, null);
Assert.IsTrue(result.IsSuccess);
Assert.AreEqual(expected: 2, actual: RuntimeStorage.Cache.Count);
var jobConfigDestinationFolder = runtimeConfigFolder?.ToString().Split("Generation_").First();
// Verify output configuration is expected
var actualConf = PropertiesDictionary.From(this.RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), "customconfiggentest1.conf")]);
var expectedConf = PropertiesDictionary.From(await File.ReadAllTextAsync(@"Resource\customJobConfig1.conf"));
var matches = PropertiesDictionary.Match(expectedConf, actualConf).ToList();
foreach (var match in matches)
{
Console.WriteLine($"prop:{match.Item1 ?? "null"}, expected:<{match.Item2 ?? "null"}>, actual:<{match.Item3 ?? "null"}>");
}
foreach (var match in matches)
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"property:{match.Item1}");
}
//verify second job conf is generated as expected
flowName = "customconfiggentest2";
actualConf = PropertiesDictionary.From(this.RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), "customconfiggentest2.conf")]);
expectedConf = PropertiesDictionary.From(await File.ReadAllTextAsync(@"Resource\customJobConfig2.conf"));
matches = PropertiesDictionary.Match(expectedConf, actualConf).ToList();
foreach (var match in matches)
{
Console.WriteLine($"prop:{match.Item1 ?? "null"}, expected:<{match.Item2 ?? "null"}>, actual:<{match.Item3 ?? "null"}>");
}
foreach (var match in matches)
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"property:{match.Item1}");
}
Cleanup();
}
public async Task TestBatchSchedule()
{
var flowName = "configgenbatchtest";
var testingConfig = await File.ReadAllTextAsync(@"Resource\batchFlow.json");
var current = DateTime.UtcNow;
var startTimeConfig = current.AddDays(-1);
var endTimeConfig = current.AddDays(1);
var normalizedStartTime = startTimeConfig.Add(-startTimeConfig.TimeOfDay);
testingConfig = testingConfig.Replace("${startTime}", startTimeConfig.ToString("o"));
testingConfig = testingConfig.Replace("${endTime}", endTimeConfig.ToString("o"));
await DesignTimeStorage.SaveByName(flowName, testingConfig, FlowDataManager.DataCollectionName);
await CommonData.Add("defaultJobTemplate", @"Resource\batchSparkJobTemplate.json");
await CommonData.Add(ConfigFlattenerManager.DefaultConfigName, @"Resource\flattenerConfig.json");
await CommonData.Add(FlowDataManager.CommonDataName_DefaultFlowConfig, @"Resource\batchFlowDefault.json");
var result = await this.RuntimeConfigGeneration.GenerateRuntimeConfigs(flowName);
var runtimeConfigFolder = result.Properties?.GetValueOrDefault(PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder, null);
Assert.IsTrue(result.IsSuccess);
Assert.AreEqual(expected: 8, actual: RuntimeStorage.Cache.Count);
var processTime = Regex.Replace(normalizedStartTime.ToString("s"), "[^0-9]", "");
var actualConf = PropertiesDictionary.From(this.RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString() + $@"/Recurring/{processTime}", $"{flowName}.conf")]);
var conf = await File.ReadAllTextAsync(@"Resource\batchJobConfig.conf");
conf = conf.Replace("${processTime}", processTime);
conf = conf.Replace("${startTime}", normalizedStartTime.AddDays(-2).ToString("o"));
conf = conf.Replace("${endTime}", normalizedStartTime.AddDays(1).AddSeconds(-1).ToString("o"));
var expectedConf = PropertiesDictionary.From(conf);
var matches = PropertiesDictionary.Match(expectedConf, actualConf).ToList();
foreach (var match in matches)
{
Console.WriteLine($"prop:{match.Item1 ?? "null"}, expected:<{match.Item2 ?? "null"}>, actual:<{match.Item3 ?? "null"}>");
}
foreach (var match in matches)
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"property:{match.Item1}");
}
Cleanup();
}
public async Task DatabricksTestEndToEndGeneration()
{
var flowName = "dbconfiggentest";
var testingConfig = await File.ReadAllTextAsync(@"Resource\databricksFlowSaved.json");
await DesignTimeStorage.SaveByName(flowName, testingConfig, FlowDataManager.DataCollectionName);
await CommonData.Add("defaultJobTemplate", @"Resource\sparkJobTemplate.json");
await CommonData.Add(ConfigFlattenerManager.DefaultConfigName, @"Resource\flattenerConfig.json");
await CommonData.Add(FlowDataManager.CommonDataName_DefaultFlowConfig, @"Resource\flowDefault.json");
var result = await this.RuntimeConfigGeneration.GenerateRuntimeConfigs(flowName);
var runtimeConfigFolder = result.Properties?.GetValueOrDefault(PrepareJobConfigVariables.ResultPropertyName_RuntimeConfigFolder, null);
Assert.IsTrue(result.IsSuccess);
Assert.AreEqual(expected: 4, actual: RuntimeStorage.Cache.Count);
// verify output schema file is expected
var expectedSchema = JsonConfig.From(await File.ReadAllTextAsync(@"Resource\schema.json"));
var actualSchema = JsonConfig.From(RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), "inputschema.json")]);
foreach (var match in JsonConfig.Match(expectedSchema, actualSchema))
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}");
}
// verify output projection file is expected
var expectedProjection = await File.ReadAllTextAsync(@"Resource\projection.txt");
var actualProjection = RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), "projection.txt")];
Assert.AreEqual(expected: expectedProjection, actual: actualProjection);
// verify transform file is exepcted
var expectedTransform = await File.ReadAllTextAsync(@"Resource\configgentest-combined.txt");
var actualTransform = RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), "dbconfiggentest-combined.txt")];
Assert.AreEqual(expected: expectedTransform, actual: actualTransform);
// Verify output configuration is expected
var actualConf = PropertiesDictionary.From(this.RuntimeStorage.Cache[ResourcePathUtil.Combine(runtimeConfigFolder.ToString(), $"{flowName}.conf")]);
var expectedConf = PropertiesDictionary.From(await File.ReadAllTextAsync(@"Resource\databricksJobConfig.conf"));
var matches = PropertiesDictionary.Match(expectedConf, actualConf).ToList();
foreach (var match in matches)
{
Console.WriteLine($"prop:{match.Item1 ?? "null"}, expected:<{match.Item2 ?? "null"}>, actual:<{match.Item3 ?? "null"}>");
}
foreach (var match in matches)
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"property:{match.Item1}");
}
// Verify metrics
var expectedConfig = JsonConfig.From(await File.ReadAllTextAsync(@"Resource\databricksFlowStarted.json"));
var actualConfig = JsonConfig.From(await this.DesignTimeStorage.GetByName(flowName, FlowDataManager.DataCollectionName));
foreach (var match in JsonConfig.Match(expectedConfig, actualConfig))
{
Assert.AreEqual(expected: match.Item2, actual: match.Item3, message: $"path:{match.Item1}");
}
Cleanup();
}
