public async Task <object> ExecuteOperations(ExecutionContext parentContext, IList <WorkflowModel.Operation> operations)
{
var cumulativeValues = parentContext.Values;
object resultOut = null;
foreach (var operation in operations)
{
var childContext = parentContext.CreateChildContext(operation, cumulativeValues);
await ExecuteOperation(childContext);
if (childContext.ValuesOut != null)
{
cumulativeValues = MergeUtils.Merge(childContext.ValuesOut, cumulativeValues);
resultOut = MergeUtils.Merge(childContext.ValuesOut, resultOut);
}
}
return(resultOut);
}
private async Task ExecuteOperation(ExecutionContext context)
{
var operation = context.Operation;
if (context.Operation.@foreach == null)
{
await ExecuteOperationInner(context);
}
else
{
var foreachContexts = new List <ExecutionContext>();
var foreachValuesInList = ProcessValuesForeachIn([email protected], context.Values);
foreach (var foreachValuesIn in foreachValuesInList)
{
var foreachContext = context.CreateChildContext(operation, MergeUtils.Merge(foreachValuesIn, context.Values));
foreachContexts.Add(foreachContext);
await ExecuteOperationInner(foreachContext);
}
var valuesOut = default(object);
if ([email protected] != null)
{
valuesOut = ProcessValuesForeachOut([email protected], foreachContexts.Select(foreachContext => foreachContext.Values).ToList());
}
else
{
foreach (var foreachValuesOut in foreachContexts.Select(foreachContext => foreachContext.ValuesOut))
{
valuesOut = MergeUtils.Merge(foreachValuesOut, valuesOut);
}
}
context.AddValuesOut(valuesOut);
}
}
private async Task ExecuteOperation(ExecutionContext context)
{
var operation = context.Operation;
if (operation.values != null)
{
context.AddValuesIn(ProcessValues(operation.values, context.Values));
}
var patternOkay = context.PatternMatcher.IsMatch(context.Path);
var message = ConvertToString(ProcessValues(operation.message, context.Values));
var conditionOkay = true;
if (!string.IsNullOrEmpty(operation.condition))
{
var conditionResult = _jmesPathQuery.Search(operation.condition, context.Values);
conditionOkay = ConditionBoolean(conditionResult);
}
for (var shouldExecute = patternOkay && conditionOkay; shouldExecute; shouldExecute = await EvaluateRepeat(context))
{
if (!string.IsNullOrEmpty(message))
{
_console.WriteLine();
_console.WriteLine($"{new string(' ', context.Indent * 2)}- {message.Color(ConsoleColor.Cyan)}");
}
var debugPath = Path.Combine(OutputDirectory.Required(), "logs", $"{++_operationCount:000}-{new string('-', context.Indent * 2)}{new string((message ?? operation.write ?? operation.request ?? operation.template ?? string.Empty).Select(ch => char.IsLetterOrDigit(ch) ? ch : '-').ToArray())}.yaml");
Directory.CreateDirectory(Path.GetDirectoryName(debugPath));
using (var writer = _secretTracker.FilterTextWriter(File.CreateText(debugPath)))
{
var logentry = new Dictionary <object, object>
{
{
"operation",
new Dictionary <object, object>
{
{ "message", message },
{ "target", operation.target },
{ "condition", operation.condition },
{ "repeat", operation.repeat },
{ "request", operation.request },
{ "template", operation.template },
{ "write", operation.write },
}
},
{ "valuesIn", context.ValuesIn },
{ "valuesOut", context.ValuesOut },
{ "request", null },
{ "response", null },
{ "cumulativeValues", context.Values },
};
try
{
object result = null;
// First special type of operation - executing a request
if (!string.IsNullOrWhiteSpace(operation.request))
{
WorkflowModel.Request request = context.TemplateEngine.Render <WorkflowModel.Request>(
operation.request,
context.Values);
logentry["request"] = request;
HttpAuthentication auth = null;
if (request.auth != null)
{
// TODO: remove these defaults
auth = new HttpAuthentication
{
tenant = request?.auth?.tenant ?? "common",
resourceId = request?.auth?.resource ?? "499b84ac-1321-427f-aa17-267ca6975798",
clientId = request?.auth?.client ?? "e8f3cc86-b3b2-4ebb-867c-9c314925b384",
interactive = IsInteractive
};
}
var client = _clientFactory.Create(auth);
var method = new HttpMethod(request.method ?? "GET");
if (IsDryRun && method.Method != "GET")
{
_console.WriteLine($"Skipping {method.Method.ToString().Color(ConsoleColor.DarkYellow)} {request.url}");
}
else
{
try
{
var jsonRequest = new JsonRequest
{
method = method,
url = request.url,
headers = request.headers,
body = request.body,
secret = request.secret,
};
var jsonResponse = await client.SendAsync(jsonRequest);
if ((int)jsonResponse.status >= 400)
{
throw new ApplicationException($"Request failed with status code {jsonResponse.status}");
}
result = new WorkflowModel.Response
{
status = (int)jsonResponse.status,
headers = jsonResponse.headers,
body = jsonResponse.body,
};
logentry["response"] = result;
}
catch
{
// TODO - retry logic here?
throw;
}
}
}
// Second special type of operation - rendering a template
if (!string.IsNullOrWhiteSpace(operation.template))
{
if (!string.IsNullOrEmpty(operation.write))
{
var targetPath = Path.Combine(OutputDirectory.Required(), operation.write);
Directory.CreateDirectory(Path.GetDirectoryName(targetPath));
using (var targetWriter = File.CreateText(targetPath))
{
if (!string.IsNullOrWhiteSpace(operation.template))
{
context.TemplateEngine.Render(operation.template, context.Values, targetWriter);
}
}
}
else
{
result = context.TemplateEngine.Render <object>(operation.template, context.Values);
}
}
// Third special type of operation - nested operations
if (operation.operations != null)
{
result = await ExecuteOperations(context, operation.operations);
}
// If output is specifically stated - use it to query
if (operation.output != null)
{
if (operation.operations != null)
{
// for nested operations, output expressions can pull in the current operation's cumulative values as well
context.AddValuesOut(ProcessValues(operation.output, MergeUtils.Merge(result, context.Values) ?? context.Values));
}
else if (result != null)
{
// for request and template operations, the current operation result is a well-known property to avoid collisions
var merged = MergeUtils.Merge(new Dictionary <object, object> {
{ "result", result }
}, context.Values);
context.AddValuesOut(ProcessValues(operation.output, merged));
}
else
{
// there are no values coming out of this operation - output queries are based only on cumulative values
context.AddValuesOut(ProcessValues(operation.output, context.Values));
}
}
if (operation.@throw != null)
{
var throwMessage = ConvertToString(ProcessValues([email protected], context.Values));
throwMessage = string.IsNullOrEmpty(throwMessage) ? message : throwMessage;
var throwDetails = ProcessValues([email protected], context.Values);
_console.WriteLine(throwMessage.Color(ConsoleColor.DarkRed));
if (throwDetails != null)
{
_console.WriteLine(_serializers.YamlSerializer.Serialize(throwDetails).Color(ConsoleColor.DarkRed));
}
throw new OperationException(string.IsNullOrEmpty(throwMessage) ? message : throwMessage)
{
Details = throwDetails
};
}
// otherwise if output is unstated, and there are nested operations with output - those flows back as effective output
if (operation.output == null && operation.operations != null && result != null)
{
context.AddValuesOut(result);
}
}
catch (Exception ex)
{
logentry["error"] = new Dictionary <object, object>
{
{ "type", ex.GetType().FullName },
{ "message", ex.Message },
{ "stack", ex.StackTrace },
};
throw;
}
finally
{
logentry["valuesIn"] = context?.ValuesIn;
logentry["valuesOut"] = context?.ValuesOut;
logentry["cumulativeValues"] = context?.Values;
_serializers.YamlSerializer.Serialize(writer, logentry);
}
}
}
}
public async Task <int> ExecuteCore(bool generateOnly)
{
if (!OutputDirectory.HasValue())
{
OutputDirectory.TryParse($"_output");
}
var logsPath = Path.Combine(OutputDirectory.Required(), "logs");
if (Directory.Exists(logsPath))
{
foreach (var file in Directory.EnumerateFiles(logsPath))
{
File.Delete(file);
}
}
var blueprint = _blueprintManager.GetBlueprintPackage(Blueprint.Required());
if (blueprint == null)
{
throw new ApplicationException($"Unable to locate blueprint {Blueprint.Required()}");
}
var templateEngine = _templateEngineFactory.Create(new TemplateEngineOptions
{
FileSystem = new BlueprintPackageFileSystem(blueprint)
});
var eachValues = new List <object>();
if (blueprint.Exists("values.yaml"))
{
using (var reader = blueprint.OpenText("values.yaml"))
{
eachValues.Add(_serializers.YamlDeserializer.Deserialize(reader));
}
}
var defaultValuesFiles =
File.Exists("atlas-values.yaml") ? new[] { "atlas-values.yaml" } :
File.Exists("values.yaml") ? new[] { "values.yaml" } :
new string[0];
foreach (var valuesFile in Values.OptionalMany(defaultValuesFiles))
{
using (var reader = File.OpenText(valuesFile))
{
eachValues.Add(_serializers.YamlDeserializer.Deserialize(reader));
}
}
if (Set.HasValue())
{
var setValues = new Dictionary <object, object>();
foreach (var set in Set.Values)
{
var parts = set.Split('=', 2);
if (parts.Length == 1)
{
throw new ApplicationException("Equal sign required when using the option --set name=value");
}
var name = parts[0];
var value = parts[1];
var segments = name.Split('.');
if (segments.Any(segment => string.IsNullOrEmpty(segment)))
{
throw new ApplicationException("Name must not have empty segments when using the option --set name=value");
}
var cursor = (IDictionary <object, object>)setValues;
foreach (var segment in segments.Reverse().Skip(1).Reverse())
{
if (cursor.TryGetValue(segment, out var child) && child is IDictionary <object, object> )
{
cursor = (IDictionary <object, object>)child;
}
else
{
child = new Dictionary <object, object>();
cursor[segment] = child;
cursor = (IDictionary <object, object>)child;
}
}
cursor[segments.Last()] = value;
}
eachValues.Add(setValues);
}
IDictionary <object, object> values = new Dictionary <object, object>();
foreach (var addValues in eachValues)
{
values = (IDictionary <object, object>)MergeUtils.Merge(addValues, values) ?? values;
}
object model;
var modelTemplate = "model.yaml";
var modelExists = blueprint.Exists(modelTemplate);
if (modelExists)
{
model = templateEngine.Render <object>(modelTemplate, values);
}
else
{
model = values;
}
var workflowTemplate = "workflow.yaml";
var workflowContents = new StringBuilder();
using (var workflowWriter = new StringWriter(workflowContents))
{
templateEngine.Render(workflowTemplate, model, workflowWriter);
}
// NOTE: the workflow is rendered BEFORE writing these output files because it may contain
// calls to the "secret" helper which will redact tokens that might have been provided in values
// write values to output folder
GenerateOutput("values.yaml", writer => _serializers.YamlSerializer.Serialize(writer, values));
if (modelExists)
{
// write normalized values to output folder
GenerateOutput(modelTemplate, writer => templateEngine.Render(modelTemplate, values, writer));
}
// write workflow to output folder
GenerateOutput("workflow.yaml", writer => writer.Write(workflowContents.ToString()));
var workflow = _serializers.YamlDeserializer.Deserialize <WorkflowModel>(new StringReader(workflowContents.ToString()));
var patternMatcher = _patternMatcherFactory.Create(Target.Values.Any() ? Target.Values : new List <string>()
{
"/**"
});
if (generateOnly == false)
{
var context = new ExecutionContext(templateEngine, patternMatcher, null);
context.AddValuesIn(ProcessValues(workflow.values, context.Values));
var resultOut = await ExecuteOperations(context, workflow.operations);
if (workflow.output != null)
{
context.AddValuesOut(ProcessValues(workflow.output, MergeUtils.Merge(resultOut, context.Values) ?? context.Values));
}
else
{
context.AddValuesOut(resultOut);
}
if (context.ValuesOut != null)
{
GenerateOutput("output.yaml", writer => _serializers.YamlSerializer.Serialize(writer, context.ValuesOut));
using (var writer = _secretTracker.FilterTextWriter(_console.Out))
{
_serializers.YamlSerializer.Serialize(writer, context.ValuesOut);
}
}
}
return(0);
}
public async Task <int> ExecuteCore(bool generateOnly)
{
if (!OutputDirectory.HasValue())
{
OutputDirectory.TryParse($"_output");
}
var logsPath = Path.Combine(OutputDirectory.Required(), "logs");
if (Directory.Exists(logsPath))
{
foreach (var file in Directory.EnumerateFiles(logsPath))
{
File.Delete(file);
}
}
var generatedPath = Path.Combine(OutputDirectory.Required(), "generated");
if (Directory.Exists(generatedPath))
{
foreach (var file in Directory.EnumerateFiles(generatedPath, "*.*", new EnumerationOptions {
RecurseSubdirectories = true
}))
{
File.Delete(file);
}
foreach (var folder in Directory.EnumerateDirectories(generatedPath))
{
Directory.Delete(folder, recursive: true);
}
}
var blueprint = await _blueprintManager.GetBlueprintPackage(Blueprint.Required());
if (blueprint == null)
{
throw new ApplicationException($"Unable to locate blueprint {Blueprint.Required()}");
}
var eachValues = new List <object>();
var defaultValuesFiles =
File.Exists("values.yaml") ? new[] { "values.yaml" } :
new string[0];
foreach (var valuesFile in Values.OptionalMany(defaultValuesFiles))
{
using (var reader = File.OpenText(valuesFile))
{
eachValues.Add(_serializers.YamlDeserializer.Deserialize(reader));
}
}
if (Set.HasValue())
{
var setValues = new Dictionary <object, object>();
foreach (var set in Set.Values)
{
var parts = set.Split('=', 2);
if (parts.Length == 1)
{
throw new ApplicationException("Equal sign required when using the option --set name=value");
}
var name = parts[0];
var value = parts[1];
var segments = name.Split('.');
if (segments.Any(segment => string.IsNullOrEmpty(segment)))
{
throw new ApplicationException("Name must not have empty segments when using the option --set name=value");
}
var cursor = (IDictionary <object, object>)setValues;
foreach (var segment in segments.Reverse().Skip(1).Reverse())
{
if (cursor.TryGetValue(segment, out var child) && child is IDictionary <object, object> )
{
cursor = (IDictionary <object, object>)child;
}
else
{
child = new Dictionary <object, object>();
cursor[segment] = child;
cursor = (IDictionary <object, object>)child;
}
}
cursor[segments.Last()] = value;
}
eachValues.Add(setValues);
}
IDictionary <object, object> values = new Dictionary <object, object>();
foreach (var addValues in eachValues)
{
values = (IDictionary <object, object>)MergeUtils.Merge(addValues, values) ?? values;
}
var(templateEngine, workflow, effectiveValues) = _workflowLoader.Load(blueprint, values, GenerateOutput);
if (generateOnly == false)
{
var patternMatcher = _patternMatcherFactory.Create(Target.Values.Any() ? Target.Values : new List <string>()
{
"/**"
});
var context = new ExecutionContext.Builder()
.UseBlueprintPackage(blueprint)
.UseTemplateEngine(templateEngine)
.UsePatternMatcher(patternMatcher)
.SetOutputDirectory(OutputDirectory.Required())
.SetNonInteractive(NonInteractive?.HasValue() ?? false)
.SetDryRun(DryRun?.HasValue() ?? false)
.SetValues(effectiveValues)
.Build();
context.AddValuesIn(_valuesEngine.ProcessValues(workflow.values, context.Values) ?? context.Values);
var resultOut = await _workflowEngine.ExecuteOperations(context, workflow.operations);
if (workflow.output != null)
{
context.AddValuesOut(_valuesEngine.ProcessValues(workflow.output, MergeUtils.Merge(resultOut, context.Values) ?? context.Values));
}
else
{
context.AddValuesOut(resultOut);
}
if (context.ValuesOut != null)
{
GenerateOutput("output.yaml", writer => _serializers.YamlSerializer.Serialize(writer, context.ValuesOut));
using (var writer = _secretTracker.FilterTextWriter(_console.Out))
{
_serializers.YamlSerializer.Serialize(writer, context.ValuesOut);
}
}
}
return(0);
}
public void BasicTests()
{
var data = MergeUtils.Merge(new[] { 1, 2, 3 }, new[] { 3, 4, 5 }, x => x);
}
private object ProcessValuesRecursive(object source, IList <object> contexts, bool promoteArrays)
{
if (source is IDictionary <object, object> sourceDictionary)
{
var arrayIsPromoting = false;
var arrayLength = 0;
var output = new Dictionary <object, object>();
foreach (var kv in sourceDictionary)
{
var result = ProcessValuesRecursive(kv.Value, contexts, promoteArrays: promoteArrays);
output[kv.Key] = result;
if (promoteArrays && result is IList <object> resultArray)
{
if (!arrayIsPromoting)
{
arrayIsPromoting = true;
arrayLength = resultArray.Count();
}
else
{
if (arrayLength != resultArray.Count())
{
throw new ApplicationException("Foreach arrays must all be same size");
}
}
}
}
if (arrayIsPromoting)
{
var arrayOutput = new List <object>();
for (var index = 0; index < arrayLength; ++index)
{
var arrayItem = output.ToDictionary(kv => kv.Key, kv => kv.Value is IList <object> valueArray ? valueArray[index] : kv.Value);
arrayOutput.Add(arrayItem);
}
return(arrayOutput);
}
return(output);
}
if (source is IList <object> sourceList)
{
return(sourceList.Select(value => ProcessValuesRecursive(value, contexts, promoteArrays: promoteArrays)).ToList());
}
if (source is string sourceString)
{
if (sourceString.StartsWith('(') && sourceString.EndsWith(')'))
{
var expression = sourceString.Substring(1, sourceString.Length - 2);
var mergedResult = default(object);
foreach (var context in contexts)
{
var result = _jmesPathQuery.Search(sourceString, context);
if (result is IList <object> resultList && mergedResult is IList <object> mergedList)
{
mergedResult = mergedList.Concat(resultList).ToList();
}
else
{
mergedResult = MergeUtils.Merge(result, mergedResult);
}
}
return(mergedResult);
}
}
return(source);
}
public void Test_TwoWayMerge_WithDefaultOpion()
{
Assert.Throws(typeof(MergeOperationException), () => MergeUtils.Merge("", "abc", "cde"),
"Should throw MergeOperationException");
}
public int Test_ResolveConflicts(int left, int right, ConflictResolutionOptions options)
{
return(MergeUtils.ResolveConflict(left, right, options));
}
public string Test_TwoWayMerge_WithOptions(string origin, string left, string right, ConflictResolutionOptions options)
{
return(MergeUtils.Merge(origin, left, right, options));
}
public void Test_TwoWayMerge_WithException()
{
Assert.Throws(typeof(MergeOperationException), () => MergeUtils.Merge("", "abc", "cde", ConflictResolutionOptions.RaiseException),
"Should throw MergeOperationException");
}
public void Test_MergeTwoValues_WithDefaultOption()
{
Assert.Throws(typeof(MergeOperationException), () => MergeUtils.Merge(1, 2), "Should throw MergeOperationException by default");
}
public void Test_MergeTwoRefValues_WithException()
{
Assert.Throws(typeof(MergeOperationException), () => MergeUtils.Merge(null, "", ConflictResolutionOptions.RaiseException),
"Should throw MergeOperationException");
}
public string Test_MergeTwoRefValues_WithOptions(string left, string right, ConflictResolutionOptions options)
{
return(MergeUtils.Merge(left, right, options));
}
public int Test_MergeTwoValues_WithOptions(int left, int right, ConflictResolutionOptions options)
{
return(MergeUtils.Merge(left, right, options));
}
public void Test_ResolveConflicts_WithException()
{
Assert.Throws(typeof(MergeOperationException),
() => MergeUtils.ResolveConflict(1, 2, ConflictResolutionOptions.RaiseException), "Should throw exception");
}