public async Task <ApiResult> GetCodeGen([FromBody] JObject config)
{
try
{
RolesCheck.EnsureWriter(Request, _isLocal);
Ensure.NotNull(config, "config");
string queries = string.Join("\n", config.GetValue("query").ToString());
var ruleDefinitionRaw = config.Value <JArray>("rules");
var ruleDefinitionList = ruleDefinitionRaw.ToObject <List <FlowGuiRule> >();
var ruleDefinitions = RuleDefinitionGenerator.GenerateRuleDefinitions(ruleDefinitionList, config.GetValue("name").ToString());
RulesCode codeGen = CodeGen.GenerateCode(queries, ruleDefinitions, config.GetValue("name").ToString());
await Task.Yield();
return(ApiResult.CreateSuccess(codeGen.Code));
}
catch (Exception e)
{
_logger.LogError(e, e.Message);
return(ApiResult.CreateError(e.Message));
}
}
public void SimpleRulesTestForDerivedTable()
{
Engine engine = new Engine();
StreamReader sr = new StreamReader("UserCodeSimRulDerTable.txt");
string code = sr.ReadToEnd();
sr.Close();
StreamReader sr2 = new StreamReader("Rules.json");
string rules = sr2.ReadToEnd();
sr2.Close();
StreamReader sr3 = new StreamReader("QueryTemplates.xml");
string queryTemplates = sr3.ReadToEnd();
sr3.Close();
StreamReader sr4 = new StreamReader("OutputTemplates.xml");
string outputTemplates = sr4.ReadToEnd();
sr4.Close();
RulesCode result = engine.GenerateCode(code, rules, queryTemplates, outputTemplates, "P5");
StreamReader sr5 = new StreamReader("CGenSimRulDerTable.txt");
string expectedCodegen = sr5.ReadToEnd();
sr5.Close();
Assert.AreEqual(result.Code, expectedCodegen, "Codegen is not as expected");
}
public void NoPivotsTest()
{
Engine engine = new Engine();
string code = "T1 = ProcessAggregateRules(DataXProcessedInput)";
StreamReader sr2 = new StreamReader("Rules.json");
string rules = sr2.ReadToEnd();
sr2.Close();
StreamReader sr3 = new StreamReader("QueryTemplates.xml");
string queryTemplates = sr3.ReadToEnd();
sr3.Close();
StreamReader sr4 = new StreamReader("OutputTemplates.xml");
string outputTemplates = sr4.ReadToEnd();
sr4.Close();
RulesCode result = engine.GenerateCode(code, rules, queryTemplates, outputTemplates, "P8");
StreamReader sr5 = new StreamReader("CGenNoPivots.txt");
string expectedCodegen = sr5.ReadToEnd();
sr5.Close();
Assert.AreEqual(result.Code, expectedCodegen, "Codegen is not as expected");
}
public void RulesandAlertsTestWithDefaultTemplate()
{
Engine engine = new Engine();
StreamReader sr = new StreamReader("usercode.txt");
string code = sr.ReadToEnd();
sr.Close();
StreamReader sr2 = new StreamReader("Rules.json");
string rules = sr2.ReadToEnd();
sr2.Close();
StreamReader sr3 = new StreamReader("QueryTemplates.xml");
string queryTemplates = sr3.ReadToEnd();
sr3.Close();
StreamReader sr4 = new StreamReader("OutputTemplates.xml");
string outputTemplates = sr4.ReadToEnd();
sr4.Close();
RulesCode result = engine.GenerateCode(code, rules, "P1");
StreamReader sr5 = new StreamReader("cgenDefault.txt");
string expectedCodegen = sr5.ReadToEnd();
sr5.Close();
Assert.AreEqual(result.Code, expectedCodegen, "Codegen is not as expected");
}
public void MixedAlertWithTickTest()
{
Engine engine = new Engine();
StreamReader sr = new StreamReader("UserCodeAggWithDot.txt");
string code = sr.ReadToEnd();
sr.Close();
StreamReader sr2 = new StreamReader("Rules.json");
string rules = sr2.ReadToEnd();
sr2.Close();
StreamReader sr3 = new StreamReader("QueryTemplates.xml");
string queryTemplates = sr3.ReadToEnd();
sr3.Close();
StreamReader sr4 = new StreamReader("OutputTemplates.xml");
string outputTemplates = sr4.ReadToEnd();
sr4.Close();
RulesCode result = engine.GenerateCode(code, rules, queryTemplates, outputTemplates, "P6.1");
StreamReader sr5 = new StreamReader("CGenMixedAlertWithTick.txt");
string expectedCodegen = sr5.ReadToEnd();
sr5.Close();
Assert.AreEqual(result.Code, expectedCodegen, "Codegen is not as expected");
}
/// <summary>
/// Generate and set the info for the transform file which will be used to generate JobConfig
/// </summary>
/// <returns></returns>
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config?.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
string queries = string.Join("\n", guiConfig.Process?.Queries);
string ruleDefinitions = RuleDefinitionGenerator.GenerateRuleDefinitions(guiConfig.Rules, config.Name);
RulesCode rulesCode = CodeGen.GenerateCode(queries, ruleDefinitions, config.Name);
Ensure.NotNull(rulesCode, "rulesCode");
// Save the rulesCode object for downstream processing
flowToDeploy.SetAttachment(AttachmentName_CodeGenObject, rulesCode);
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var secretName = $"{config.Name}-transform";
Configuration.TryGet(Constants.ConfigSettingName_SparkType, out string sparkType);
var uriPrefix = KeyVaultClient.GetUriPrefix(sparkType);
var transformFileSecret = SecretUriParser.ComposeUri(runtimeKeyVaultName, secretName, uriPrefix);
flowToDeploy.SetStringToken(TokenName_TransformFile, transformFileSecret);
await Task.CompletedTask;
return("done");
}
public override async Task <string> Process(FlowDeploymentSession flowToDeploy)
{
var config = flowToDeploy.Config;
var guiConfig = config?.GetGuiConfig();
if (guiConfig == null)
{
return("no gui input, skipped.");
}
string queries = string.Join("\n", guiConfig.Process?.Queries);
string ruleDefinitions = RuleDefinitionGenerator.GenerateRuleDefinitions(guiConfig.Rules, config.Name);
RulesCode rulesCode = CodeGen.GenerateCode(queries, ruleDefinitions, config.Name);
Ensure.NotNull(rulesCode, "rulesCode");
// Save the rulesCode object for downstream processing
flowToDeploy.SetAttachment(AttachmentName_CodeGenObject, rulesCode);
var runtimeConfigBaseFolder = flowToDeploy.GetTokenString(PrepareJobConfigVariables.TokenName_RuntimeConfigFolder);
Ensure.NotNull(runtimeConfigBaseFolder, "runtimeConfigBaseFolder");
var runtimeKeyVaultName = flowToDeploy.GetTokenString(PortConfigurationSettings.TokenName_RuntimeKeyVaultName);
Ensure.NotNull(runtimeKeyVaultName, "runtimeKeyVaultName");
var filePath = ResourcePathUtil.Combine(runtimeConfigBaseFolder, $"{config.Name}-combined.txt");
var transformFilePath = await RuntimeStorage.SaveFile(filePath, rulesCode.Code);
var secretName = $"{config.Name}-transform";
var transformFileSecret = await KeyVaultClient.SaveSecretAsync(runtimeKeyVaultName, secretName, transformFilePath);
flowToDeploy.SetStringToken(TokenName_TransformFile, transformFileSecret);
return("done");
}
private async Task <FlowOutputSpec[]> ProcessOutputs(FlowGuiOutput[] uiOutputs, RulesCode rulesCode, string configName)
{
var outputList = uiOutputs.Select(o => o.Id).ToList();
var outputsFiltered = rulesCode.Outputs.Where(o => outputList.Contains(o.Item2)).ToList();
var flattenedOutputs = new Dictionary <string, List <string> >();
foreach (var o in outputsFiltered)
{
var outputNames = o.Item1.Split(new char[] { ',' });
foreach (string outputName in outputNames)
{
var name = outputName.Trim();
if (flattenedOutputs.TryGetValue(name, out List <string> val))
{
val.Add(o.Item2);
}
else
{
flattenedOutputs[name] = new List <string>()
{
o.Item2
};
}
}
}
List <FlowOutputSpec> fOutputList = new List <FlowOutputSpec>();
foreach (var fOut in flattenedOutputs)
{
FlowOutputSpec flowOutput = new FlowOutputSpec()
{
Name = fOut.Key
};
foreach (var item in fOut.Value)
{
var output = uiOutputs.SingleOrDefault(o => o.Id.Equals(item));
switch (output.Type.ToLower())
{
case "cosmosdb":
{
var cosmosDbOutput = ProcessOutputCosmosDb(output);
Ensure.EnsureNullElseThrowNotSupported(flowOutput.CosmosDbOutput, "Multiple target cosmosDB ouptut for same dataset not supported.");
flowOutput.CosmosDbOutput = cosmosDbOutput;
break;
}
case "eventhub":
{
var eventhubOutput = ProcessOutputEventHub(output);
Ensure.EnsureNullElseThrowNotSupported(flowOutput.EventHubOutput, "Multiple target eventHub/metric ouptut for same dataset not supported.");
flowOutput.EventHubOutput = eventhubOutput;
break;
}
case "metric":
{
if (LocalUtility.IsLocalEnabled(Configuration))
{
if (Configuration.TryGet(Constants.ConfigSettingName_LocalMetricsHttpEndpoint, out string localMetricsEndpoint))
{
var httpOutput = ProcessLocalOutputMetric(configName, localMetricsEndpoint);
Ensure.EnsureNullElseThrowNotSupported(flowOutput.HttpOutput, "Multiple target httpost/metric ouptut for same dataset not supported.");
flowOutput.HttpOutput = httpOutput;
}
break;
}
else
{
var eventhubOutput = ProcessOutputMetric(output);
Ensure.EnsureNullElseThrowNotSupported(flowOutput.EventHubOutput, "Multiple target eventHub/metric ouptut for same dataset not supported.");
flowOutput.EventHubOutput = eventhubOutput;
break;
}
}
case "blob":
{
var blobOutput = await ProcessOutputBlob(configName, output);
Ensure.EnsureNullElseThrowNotSupported(flowOutput.BlobOutput, "Multiple target blob ouptut for same dataset not supported.");
flowOutput.BlobOutput = blobOutput;
break;
}
case "local":
{
var blobOutput = ProcessOutputLocal(configName, output);
Ensure.EnsureNullElseThrowNotSupported(flowOutput.BlobOutput, "Multiple target blob ouptut for same dataset not supported.");
flowOutput.BlobOutput = blobOutput;
break;
}
default:
throw new NotSupportedException($"{output.Type} output type not supported");
}
}
fOutputList.Add(flowOutput);
}
return(fOutputList.ToArray());
}
