private object MergeError(Exception exception, object context) { var yaml = _serializers.YamlSerializer.Serialize(new { error = exception }); var error = _serializers.YamlDeserializer.Deserialize <object>($@" {yaml} type: name: {exception.GetType().Name} fullName: {exception.GetType().FullName} "); var mergedContext = MergeUtils.Merge(error, context); return(mergedContext); }
public async Task <object> ExecuteOperations(OperationContext parentContext, IList <WorkflowModel.Operation> operations) { var cumulativeValues = parentContext.Values; object resultOut = null; foreach (var operation in operations) { var childContext = parentContext.CreateChildContext(operation, cumulativeValues); await ExecuteOperation(childContext); if (childContext.ValuesOut != null) { cumulativeValues = MergeUtils.Merge(childContext.ValuesOut, cumulativeValues); resultOut = MergeUtils.Merge(childContext.ValuesOut, resultOut); } } return(resultOut); }
private async Task ExecuteOperation(OperationContext context) { var operation = context.Operation; if (context.Operation.@foreach == null) { await ExecuteOperationInner(context); } else { var foreachContexts = new List <OperationContext>(); var foreachValuesInList = _valuesEngine.ProcessValuesForeachIn([email protected], context.Values); foreach (var foreachValuesIn in foreachValuesInList) { var foreachContext = context.CreateChildContext(operation, MergeUtils.Merge(foreachValuesIn, context.Values)); foreachContexts.Add(foreachContext); await ExecuteOperationInner(foreachContext); } var valuesOut = default(object); if ([email protected] != null) { valuesOut = _valuesEngine.ProcessValuesForeachOut([email protected], foreachContexts.Select(foreachContext => foreachContext.Values).ToList()); } else { foreach (var foreachValuesOut in foreachContexts.Select(foreachContext => foreachContext.ValuesOut)) { valuesOut = MergeUtils.Merge(foreachValuesOut, valuesOut); } } context.AddValuesOut(valuesOut); } }
private async Task ExecuteOperationInner(OperationContext context) { var operation = context.Operation; if (operation.values != null) { context.AddValuesIn(_valuesEngine.ProcessValues(operation.values, context.Values)); } var patternOkay = context.PatternMatcher.IsMatch(context.Path); var conditionOkay = operation.condition == null ? true : _valuesEngine.EvaluateToBoolean(operation.condition, context.Values); for (var shouldExecute = patternOkay && conditionOkay; shouldExecute; shouldExecute = await EvaluateRepeat(context)) { var message = _valuesEngine.EvaluateToString(operation.message, context.Values); var write = _valuesEngine.EvaluateToString(operation.write, context.Values); var workflow = _valuesEngine.EvaluateToString(operation.workflow, context.Values); if (!string.IsNullOrEmpty(message)) { _console.WriteLine(); _console.WriteLine($"{new string(' ', context.Indent * 2)}- {message.Color(ConsoleColor.Cyan)}"); } var debugPath = Path.Combine(context.ExecutionContext.OutputDirectory, "logs", $"{++_operationCount:000}-{new string('-', context.Indent * 2)}{new string((message ?? write ?? operation.request ?? operation.template ?? workflow ?? string.Empty).Select(ch => char.IsLetterOrDigit(ch) ? ch : '-').ToArray())}.yaml"); Directory.CreateDirectory(Path.GetDirectoryName(debugPath)); using (var writer = _secretTracker.FilterTextWriter(File.CreateText(debugPath))) { var logentry = new Dictionary <object, object> { { "operation", new Dictionary <object, object> { { "message", message }, { "target", operation.target }, { "condition", operation.condition }, { "repeat", operation.repeat }, { "request", operation.request }, { "template", operation.template }, { "workflow", workflow }, { "write", write }, } }, { "valuesIn", context.ValuesIn }, { "valuesOut", context.ValuesOut }, { "request", null }, { "response", null }, { "cumulativeValues", context.Values }, }; try { // object result = null; object outputContext = context.Values; try { // First special type of operation - executing a request if (!string.IsNullOrWhiteSpace(operation.request)) { var request = context.TemplateEngine.Render <WorkflowModel.Request>( operation.request, context.Values); logentry["request"] = request; HttpAuthentication auth = null; if (request.auth != null) { // TODO: remove these defaults auth = new HttpAuthentication { tenant = request?.auth?.tenant ?? "common", resourceId = request?.auth?.resource ?? "499b84ac-1321-427f-aa17-267ca6975798", clientId = request?.auth?.client ?? "e8f3cc86-b3b2-4ebb-867c-9c314925b384", interactive = context.ExecutionContext.IsInteractive }; } var client = _clientFactory.Create(auth); var method = new HttpMethod(request.method ?? "GET"); var parts = UriParts.Parse(request.url); foreach (var query in request.query ?? Enumerable.Empty <KeyValuePair <string, object> >()) { parts.Query = parts.Query.Add(query.Key, Convert.ToString(query.Value)); } var url = parts.ToString(); if (context.ExecutionContext.IsDryRun && method.Method != "GET") { _console.WriteLine($"Skipping {method.Method.ToString().Color(ConsoleColor.DarkYellow)} {request.url}"); } else { var jsonRequest = new JsonRequest { method = method, url = url, headers = request.headers, body = request.body, secret = request.secret, }; var jsonResponse = await client.SendAsync(jsonRequest); var response = new WorkflowModel.Response { status = (int)jsonResponse.status, headers = jsonResponse.headers, body = jsonResponse.body, }; logentry["response"] = response; outputContext = MergeUtils.Merge(new Dictionary <object, object> { { "result", response } }, outputContext); if (response.status >= 400) { var error = new RequestException($"Request failed with status code {jsonResponse.status}") { Request = request, Response = response, }; throw error; } } } // Second special type of operation - rendering a template if (!string.IsNullOrWhiteSpace(operation.template)) { if (!string.IsNullOrEmpty(write)) { if (string.Equals(write, "stdout")) { context.TemplateEngine.Render(operation.template, context.Values, Console.Out); } else if (string.Equals(write, "stderr")) { context.TemplateEngine.Render(operation.template, context.Values, Console.Error); } else { var targetPath = Path.Combine(context.ExecutionContext.OutputDirectory, write); Directory.CreateDirectory(Path.GetDirectoryName(targetPath)); using (var targetWriter = File.CreateText(targetPath)) { context.TemplateEngine.Render(operation.template, context.Values, targetWriter); } } } if (operation.output != null) { var templateResult = context.TemplateEngine.Render <object>(operation.template, context.Values); outputContext = MergeUtils.Merge(new Dictionary <object, object> { { "result", templateResult } }, outputContext); } } if (!string.IsNullOrEmpty(workflow)) { var subBlueprint = await _blueprintManager.GetBlueprintPackageDependency(context.ExecutionContext.BlueprintPackage, workflow); if (subBlueprint == null) { throw new OperationException($"Unable to load sub-workflow {workflow}"); } var(subTemplateEngine, subWorkflow, subValues) = _workflowLoader.Load(subBlueprint, context.ValuesIn, GenerateOutput); var subContext = new ExecutionContext.Builder() .CopyFrom(context) .UseBlueprintPackage(subBlueprint) .UseTemplateEngine(subTemplateEngine) .SetValues(subValues) .Build(); if (subWorkflow.values != null) { subContext.AddValuesIn(_valuesEngine.ProcessValues(subWorkflow.values, subContext.Values)); } var nestedResult = await ExecuteOperations(subContext, subWorkflow.operations); if (subWorkflow.output != null) { nestedResult = _valuesEngine.ProcessValues(subWorkflow.output, nestedResult); } outputContext = MergeUtils.Merge(new Dictionary <object, object> { { "result", nestedResult } }, outputContext); } // Third special type of operation - nested operations if (operation.operations != null) { var nestedResult = await ExecuteOperations(context, operation.operations); if (operation.output == null) { // if output is unstated, and there are nested operations with output - those flows back as effective output context.AddValuesOut(nestedResult); } else { // if output is stated, nested operations with output are visible to output queries outputContext = MergeUtils.Merge(nestedResult, context.Values) ?? context.Values; } } // If output is specifically stated - use it to query if (operation.output != null) { context.AddValuesOut(ProcessValues(operation.output, outputContext)); } if (operation.@throw != null) { var throwMessage = _valuesEngine.EvaluateToString([email protected], context.Values); throwMessage = string.IsNullOrEmpty(throwMessage) ? message : throwMessage; var throwDetails = ProcessValues([email protected], context.Values); _console.WriteLine(throwMessage.Color(ConsoleColor.DarkRed)); if (throwDetails != null) { _console.WriteLine(_serializers.YamlSerializer.Serialize(throwDetails).Color(ConsoleColor.DarkRed)); } throw new OperationException(string.IsNullOrEmpty(throwMessage) ? message : throwMessage) { Details = throwDetails }; } } catch (Exception ex) when(CatchCondition(ex, operation.@catch, outputContext)) { if ([email protected] != null) { var mergedContext = MergeError(ex, outputContext); var catchDetails = ProcessValues([email protected], mergedContext); context.AddValuesOut(catchDetails); } } } catch (Exception ex) { logentry["error"] = new Dictionary <object, object> { { "type", ex.GetType().FullName }, { "message", ex.Message }, { "stack", ex.StackTrace }, }; throw; } finally { logentry["valuesIn"] = context?.ValuesIn; logentry["valuesOut"] = context?.ValuesOut; logentry["cumulativeValues"] = context?.Values; _serializers.YamlSerializer.Serialize(writer, logentry); } } } }
public void AddValuesOut(object valuesOut) { ValuesOut = valuesOut; Values = MergeUtils.Merge(ValuesOut, Values); }
public void AddValuesIn(object valuesIn) { ValuesIn = valuesIn; Values = MergeUtils.Merge(ValuesIn, Values); }
public (ITemplateEngine templateEngine, WorkflowModel workflow, object effectiveValues) Load(IBlueprintPackage blueprint, object values, Action <string, Action <TextWriter> > generateOutput) { var templateEngine = _templateEngineFactory.Create(new TemplateEngineOptions { FileSystem = new BlueprintPackageFileSystem(blueprint) }); var providedValues = values; if (blueprint.Exists("values.yaml")) { using (var reader = blueprint.OpenText("values.yaml")) { var defaultValues = _serializers.YamlDeserializer.Deserialize(reader); if (values == null) { values = defaultValues; } else { values = MergeUtils.Merge(values, defaultValues); } } } var premodelValues = values; if (blueprint.Exists("model.yaml")) { var model = templateEngine.Render <object>("model.yaml", values); if (model != null) { values = MergeUtils.Merge(model, values); } } var workflowContents = new StringBuilder(); using (var workflowWriter = new StringWriter(workflowContents)) { templateEngine.Render("workflow.yaml", values, workflowWriter); } // NOTE: the workflow is rendered BEFORE writing these output files because it may contain // calls to the "secret" helper which will redact tokens that might have been provided in values // write values to output folder generateOutput("values.yaml", writer => _serializers.YamlSerializer.Serialize(writer, values)); // write workflow to output folder generateOutput("workflow.yaml", writer => writer.Write(workflowContents.ToString())); var workflow = _serializers.YamlDeserializer.Deserialize <WorkflowModel>(new StringReader(workflowContents.ToString())); foreach (var generatedFile in blueprint.GetGeneratedPaths()) { using (var generatedContent = blueprint.OpenText(generatedFile)) { generateOutput($"generated/{generatedFile}", writer => writer.Write(generatedContent.ReadToEnd())); } } return(templateEngine, workflow, values); }
private object ProcessValuesRecursive(object source, IList <object> contexts, bool promoteArrays) { if (source is IDictionary <object, object> sourceDictionary) { var arrayIsPromoting = false; var arrayLength = 0; void CheckArrayIsPromoting(object result) { if (promoteArrays && result is IList <object> resultArray) { if (!arrayIsPromoting) { arrayIsPromoting = true; arrayLength = resultArray.Count(); } else { if (arrayLength != resultArray.Count()) { throw new ApplicationException("Foreach arrays must all be same size"); } } } } var output = new Dictionary <object, object>(); foreach (var kv in sourceDictionary) { var propertyName = Convert.ToString(kv.Key, CultureInfo.InvariantCulture); if (propertyName.StartsWith('(') && propertyName.EndsWith(')')) { var propertyGroupings = contexts .Select(eachContext => { var eachPropertyName = _jmesPathQuery.Search(propertyName, eachContext); return(eachPropertyName, eachContext); }) .ToList() .GroupBy(x => x.eachPropertyName, x => x.eachContext); foreach (var propertyGrouping in propertyGroupings) { var result = ProcessValuesRecursive(kv.Value, propertyGrouping.ToList(), promoteArrays: promoteArrays); output[propertyGrouping.Key] = result; CheckArrayIsPromoting(result); } } else { var result = ProcessValuesRecursive(kv.Value, contexts, promoteArrays: promoteArrays); output[propertyName] = result; CheckArrayIsPromoting(result); } } if (arrayIsPromoting) { var arrayOutput = new List <object>(); for (var index = 0; index < arrayLength; ++index) { var arrayItem = output.ToDictionary(kv => kv.Key, kv => kv.Value is IList <object> valueArray ? valueArray[index] : kv.Value); arrayOutput.Add(arrayItem); } return(arrayOutput); } return(output); } if (source is IList <object> sourceList) { return(sourceList.Select(value => ProcessValuesRecursive(value, contexts, promoteArrays: promoteArrays)).ToList()); } if (source is string sourceString) { if (sourceString.StartsWith('(') && sourceString.EndsWith(')')) { var mergedResult = default(object); foreach (var context in contexts) { var result = _jmesPathQuery.Search(sourceString, context); if (result is IList <object> resultList && mergedResult is IList <object> mergedList) { mergedResult = mergedList.Concat(resultList).ToList(); } else { mergedResult = MergeUtils.Merge(result, mergedResult); } } return(mergedResult); } } return(source); }