public static PipelineTemplateEvaluator ToPipelineTemplateEvaluator(this IExecutionContext context)
{
var templateTrace = context.ToTemplateTraceWriter();
var schema = new PipelineTemplateSchemaFactory().CreateSchema();
return(new PipelineTemplateEvaluator(templateTrace, schema, context.FileTable));
}
private PipelineTemplateEvaluator CreateTemplateEvaluator(IExecutionContext executionContext)
{
var templateTrace = executionContext.ToTemplateTraceWriter();
var schema = new PipelineTemplateSchemaFactory().CreateSchema();
return(new PipelineTemplateEvaluator(templateTrace, schema));
}
public static PipelineTemplateEvaluator ToPipelineTemplateEvaluator(this IExecutionContext context, ObjectTemplating.ITraceWriter traceWriter = null)
{
if (traceWriter == null)
{
traceWriter = context.ToTemplateTraceWriter();
}
var schema = PipelineTemplateSchemaFactory.GetSchema();
return(new PipelineTemplateEvaluator(traceWriter, schema, context.Global.FileTable));
}
public static PipelineTemplateEvaluator ToPipelineTemplateEvaluator(this IExecutionContext context, ObjectTemplating.ITraceWriter traceWriter = null)
{
if (traceWriter == null)
{
traceWriter = context.ToTemplateTraceWriter();
}
var schema = PipelineTemplateSchemaFactory.GetSchema();
return(new PipelineTemplateEvaluator(traceWriter, schema, context.Global.FileTable)
{
MaxErrorMessageLength = int.MaxValue, // Don't truncate error messages otherwise we might not scrub secrets correctly
});
}
public async Task RunAsync()
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
ArgUtil.NotNull(Action, nameof(Action));
var taskManager = HostContext.GetService <IActionManager>();
var handlerFactory = HostContext.GetService <IHandlerFactory>();
// Load the task definition and choose the handler.
Definition definition = taskManager.LoadAction(ExecutionContext, Action);
ArgUtil.NotNull(definition, nameof(definition));
ActionExecutionData handlerData = definition.Data?.Execution;
ArgUtil.NotNull(handlerData, nameof(handlerData));
// The action has post cleanup defined.
// we need to create timeline record for them and add them to the step list that StepRunner is using
if (handlerData.HasCleanup && Stage == ActionRunStage.Main)
{
string postDisplayName = null;
if (this.DisplayName.StartsWith(PipelineTemplateConstants.RunDisplayPrefix))
{
postDisplayName = $"Post {this.DisplayName.Substring(PipelineTemplateConstants.RunDisplayPrefix.Length)}";
}
else
{
postDisplayName = $"Post {this.DisplayName}";
}
var repositoryReference = Action.Reference as RepositoryPathReference;
var pathString = string.IsNullOrEmpty(repositoryReference.Path) ? string.Empty : $"/{repositoryReference.Path}";
var repoString = string.IsNullOrEmpty(repositoryReference.Ref) ? $"{repositoryReference.Name}{pathString}" :
$"{repositoryReference.Name}{pathString}@{repositoryReference.Ref}";
ExecutionContext.Debug($"Register post job cleanup for action: {repoString}");
var actionRunner = HostContext.CreateService <IActionRunner>();
actionRunner.Action = Action;
actionRunner.Stage = ActionRunStage.Post;
actionRunner.Condition = handlerData.CleanupCondition;
actionRunner.DisplayName = postDisplayName;
ExecutionContext.RegisterPostJobStep($"{actionRunner.Action.Name}_post", actionRunner);
}
IStepHost stepHost = HostContext.CreateService <IDefaultStepHost>();
// Makes directory for event_path data
var tempDirectory = HostContext.GetDirectory(WellKnownDirectory.Temp);
var workflowDirectory = Path.Combine(tempDirectory, "_github_workflow");
Directory.CreateDirectory(workflowDirectory);
var gitHubEvent = ExecutionContext.GetGitHubContext("event");
// adds the GitHub event path/file if the event exists
if (gitHubEvent != null)
{
var workflowFile = Path.Combine(workflowDirectory, "event.json");
Trace.Info($"Write event payload to {workflowFile}");
File.WriteAllText(workflowFile, gitHubEvent, new UTF8Encoding(false));
ExecutionContext.SetGitHubContext("event_path", workflowFile);
}
// Setup container stephost for running inside the container.
if (ExecutionContext.Container != null)
{
// Make sure required container is already created.
ArgUtil.NotNullOrEmpty(ExecutionContext.Container.ContainerId, nameof(ExecutionContext.Container.ContainerId));
var containerStepHost = HostContext.CreateService <IContainerStepHost>();
containerStepHost.Container = ExecutionContext.Container;
stepHost = containerStepHost;
}
// Load the inputs.
ExecutionContext.Debug("Loading inputs");
var templateTrace = ExecutionContext.ToTemplateTraceWriter();
var schema = new PipelineTemplateSchemaFactory().CreateSchema();
var templateEvaluator = new PipelineTemplateEvaluator(templateTrace, schema);
var inputs = templateEvaluator.EvaluateStepInputs(Action.Inputs, ExecutionContext.ExpressionValues);
foreach (KeyValuePair <string, string> input in inputs)
{
string message = "";
if (definition.Data?.Deprecated?.TryGetValue(input.Key, out message) == true)
{
ExecutionContext.Warning(String.Format("Input '{0}' has been deprecated with message: {1}", input.Key, message));
}
}
// Merge the default inputs from the definition
if (definition.Data?.Inputs != null)
{
var manifestManager = HostContext.GetService <IActionManifestManager>();
foreach (var input in (definition.Data?.Inputs))
{
string key = input.Key.AssertString("action input name").Value;
if (!inputs.ContainsKey(key))
{
var evaluateContext = new Dictionary <string, PipelineContextData>(StringComparer.OrdinalIgnoreCase);
foreach (var data in ExecutionContext.ExpressionValues)
{
evaluateContext[data.Key] = data.Value;
}
inputs[key] = manifestManager.EvaluateDefaultInput(ExecutionContext, key, input.Value, evaluateContext);
}
}
}
// Load the action environment.
ExecutionContext.Debug("Loading env");
var environment = new Dictionary <String, String>(VarUtil.EnvironmentVariableKeyComparer);
// Apply environment set using ##[set-env] first since these are job level env
foreach (var env in ExecutionContext.EnvironmentVariables)
{
environment[env.Key] = env.Value ?? string.Empty;
}
// Apply action's env block later.
var actionEnvironment = templateEvaluator.EvaluateStepEnvironment(Action.Environment, ExecutionContext.ExpressionValues, VarUtil.EnvironmentVariableKeyComparer);
foreach (var env in actionEnvironment)
{
environment[env.Key] = env.Value ?? string.Empty;
}
// Apply action's intra-action state at last
foreach (var state in ExecutionContext.IntraActionState)
{
environment[$"STATE_{state.Key}"] = state.Value ?? string.Empty;
}
// Create the handler.
IHandler handler = handlerFactory.Create(
ExecutionContext,
Action.Reference,
stepHost,
handlerData,
inputs,
environment,
ExecutionContext.Variables,
actionDirectory: definition.Directory);
// Print out action details
handler.PrintActionDetails(Stage);
// Run the task.
await handler.RunAsync(Stage);
}
private string GenerateDisplayName(ActionStep action, DictionaryContextData contextData, IExecutionContext context, out bool didFullyEvaluate)
{
ArgUtil.NotNull(context, nameof(context));
ArgUtil.NotNull(action, nameof(action));
var displayName = string.Empty;
var prefix = string.Empty;
var tokenToParse = default(ScalarToken);
didFullyEvaluate = false;
// Get the token we need to parse
// It could be passed in as the Display Name, or we have to pull it from various parts of the Action.
if (action.DisplayNameToken != null)
{
tokenToParse = action.DisplayNameToken as ScalarToken;
}
else if (action.Reference?.Type == ActionSourceType.Repository)
{
prefix = PipelineTemplateConstants.RunDisplayPrefix;
var repositoryReference = action.Reference as RepositoryPathReference;
var pathString = string.IsNullOrEmpty(repositoryReference.Path) ? string.Empty : $"/{repositoryReference.Path}";
var repoString = string.IsNullOrEmpty(repositoryReference.Ref) ? $"{repositoryReference.Name}{pathString}" :
$"{repositoryReference.Name}{pathString}@{repositoryReference.Ref}";
tokenToParse = new StringToken(null, null, null, repoString);
}
else if (action.Reference?.Type == ActionSourceType.ContainerRegistry)
{
prefix = PipelineTemplateConstants.RunDisplayPrefix;
var containerReference = action.Reference as ContainerRegistryReference;
tokenToParse = new StringToken(null, null, null, containerReference.Image);
}
else if (action.Reference?.Type == ActionSourceType.Script)
{
prefix = PipelineTemplateConstants.RunDisplayPrefix;
var inputs = action.Inputs.AssertMapping(null);
foreach (var pair in inputs)
{
var propertyName = pair.Key.AssertString($"{PipelineTemplateConstants.Steps}");
if (string.Equals(propertyName.Value, "script", StringComparison.OrdinalIgnoreCase))
{
tokenToParse = pair.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Run}");
break;
}
}
}
else
{
context.Error($"Encountered an unknown action reference type when evaluating the display name: {action.Reference?.Type}");
return(displayName);
}
// If we have nothing to parse, abort
if (tokenToParse == null)
{
return(displayName);
}
// Try evaluating fully
var schema = new PipelineTemplateSchemaFactory().CreateSchema();
var templateEvaluator = new PipelineTemplateEvaluator(context.ToTemplateTraceWriter(), schema);
try
{
didFullyEvaluate = templateEvaluator.TryEvaluateStepDisplayName(tokenToParse, contextData, out displayName);
}
catch (TemplateValidationException e)
{
context.Warning($"Encountered an error when evaluating display name {tokenToParse.ToString()}. {e.Message}");
return(displayName);
}
// Default to a prettified token if we could not evaluate
if (!didFullyEvaluate)
{
displayName = tokenToParse.ToDisplayString();
}
displayName = FormatStepName(prefix, displayName);
return(displayName);
}
// StepsRunner should never throw exception to caller
public async Task RunAsync(IExecutionContext jobContext)
{
ArgUtil.NotNull(jobContext, nameof(jobContext));
ArgUtil.NotNull(jobContext.JobSteps, nameof(jobContext.JobSteps));
// TaskResult:
// Abandoned (Server set this.)
// Canceled
// Failed
// Skipped
// Succeeded
CancellationTokenRegistration?jobCancelRegister = null;
jobContext.JobContext.Status = (jobContext.Result ?? TaskResult.Succeeded).ToActionResult();
var scopeInputs = new Dictionary <string, PipelineContextData>(StringComparer.OrdinalIgnoreCase);
bool checkPostJobActions = false;
while (jobContext.JobSteps.Count > 0 || !checkPostJobActions)
{
if (jobContext.JobSteps.Count == 0 && !checkPostJobActions)
{
checkPostJobActions = true;
while (jobContext.PostJobSteps.TryPop(out var postStep))
{
jobContext.JobSteps.Enqueue(postStep);
}
continue;
}
var step = jobContext.JobSteps.Dequeue();
IStep nextStep = null;
if (jobContext.JobSteps.Count > 0)
{
nextStep = jobContext.JobSteps.Peek();
}
Trace.Info($"Processing step: DisplayName='{step.DisplayName}'");
ArgUtil.NotNull(step.ExecutionContext, nameof(step.ExecutionContext));
ArgUtil.NotNull(step.ExecutionContext.Variables, nameof(step.ExecutionContext.Variables));
// Start
step.ExecutionContext.Start();
// Initialize scope
if (InitializeScope(step, scopeInputs))
{
// Populate env context for each step
Trace.Info("Initialize Env context for step");
#if OS_WINDOWS
var envContext = new DictionaryContextData();
#else
var envContext = new CaseSensitiveDictionaryContextData();
#endif
step.ExecutionContext.ExpressionValues["env"] = envContext;
foreach (var pair in step.ExecutionContext.EnvironmentVariables)
{
envContext[pair.Key] = new StringContextData(pair.Value ?? string.Empty);
}
if (step is IActionRunner actionStep)
{
// Set GITHUB_ACTION
step.ExecutionContext.SetGitHubContext("action", actionStep.Action.Name);
// Evaluate and merge action's env block to env context
var templateTrace = step.ExecutionContext.ToTemplateTraceWriter();
var schema = new PipelineTemplateSchemaFactory().CreateSchema();
var templateEvaluator = new PipelineTemplateEvaluator(templateTrace, schema);
var actionEnvironment = templateEvaluator.EvaluateStepEnvironment(actionStep.Action.Environment, step.ExecutionContext.ExpressionValues, VarUtil.EnvironmentVariableKeyComparer);
foreach (var env in actionEnvironment)
{
envContext[env.Key] = new StringContextData(env.Value ?? string.Empty);
}
}
var expressionManager = HostContext.GetService <IExpressionManager>();
try
{
// Register job cancellation call back only if job cancellation token not been fire before each step run
if (!jobContext.CancellationToken.IsCancellationRequested)
{
// Test the condition again. The job was canceled after the condition was originally evaluated.
jobCancelRegister = jobContext.CancellationToken.Register(() =>
{
// mark job as cancelled
jobContext.Result = TaskResult.Canceled;
jobContext.JobContext.Status = jobContext.Result?.ToActionResult();
step.ExecutionContext.Debug($"Re-evaluate condition on job cancellation for step: '{step.DisplayName}'.");
ConditionResult conditionReTestResult;
if (HostContext.RunnerShutdownToken.IsCancellationRequested)
{
step.ExecutionContext.Debug($"Skip Re-evaluate condition on runner shutdown.");
conditionReTestResult = false;
}
else
{
try
{
conditionReTestResult = expressionManager.Evaluate(step.ExecutionContext, step.Condition, hostTracingOnly: true);
}
catch (Exception ex)
{
// Cancel the step since we get exception while re-evaluate step condition.
Trace.Info("Caught exception from expression when re-test condition on job cancellation.");
step.ExecutionContext.Error(ex);
conditionReTestResult = false;
}
}
if (!conditionReTestResult.Value)
{
// Cancel the step.
Trace.Info("Cancel current running step.");
step.ExecutionContext.CancelToken();
}
});
}
else
{
if (jobContext.Result != TaskResult.Canceled)
{
// mark job as cancelled
jobContext.Result = TaskResult.Canceled;
jobContext.JobContext.Status = jobContext.Result?.ToActionResult();
}
}
// Evaluate condition.
step.ExecutionContext.Debug($"Evaluating condition for step: '{step.DisplayName}'");
Exception conditionEvaluateError = null;
ConditionResult conditionResult;
if (HostContext.RunnerShutdownToken.IsCancellationRequested)
{
step.ExecutionContext.Debug($"Skip evaluate condition on runner shutdown.");
conditionResult = false;
}
else
{
try
{
conditionResult = expressionManager.Evaluate(step.ExecutionContext, step.Condition);
}
catch (Exception ex)
{
Trace.Info("Caught exception from expression.");
Trace.Error(ex);
conditionResult = false;
conditionEvaluateError = ex;
}
}
// no evaluate error but condition is false
if (!conditionResult.Value && conditionEvaluateError == null)
{
// Condition == false
Trace.Info("Skipping step due to condition evaluation.");
CompleteStep(step, nextStep, TaskResult.Skipped, resultCode: conditionResult.Trace);
}
else if (conditionEvaluateError != null)
{
// fail the step since there is an evaluate error.
step.ExecutionContext.Error(conditionEvaluateError);
CompleteStep(step, nextStep, TaskResult.Failed);
}
else
{
// Run the step.
await RunStepAsync(step, jobContext.CancellationToken);
CompleteStep(step, nextStep);
}
}
finally
{
if (jobCancelRegister != null)
{
jobCancelRegister?.Dispose();
jobCancelRegister = null;
}
}
}
// Update the job result.
if (step.ExecutionContext.Result == TaskResult.Failed)
{
Trace.Info($"Update job result with current step result '{step.ExecutionContext.Result}'.");
jobContext.Result = TaskResultUtil.MergeTaskResults(jobContext.Result, step.ExecutionContext.Result.Value);
jobContext.JobContext.Status = jobContext.Result?.ToActionResult();
}
else
{
Trace.Info($"No need for updating job result with current step result '{step.ExecutionContext.Result}'.");
}
Trace.Info($"Current state: job state = '{jobContext.Result}'");
}
}
// Download all required actions.
// Make sure all condition inputs are valid.
// Build up three list of steps for jobrunner (pre-job, job, post-job).
public async Task <List <IStep> > InitializeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message)
{
Trace.Entering();
ArgUtil.NotNull(jobContext, nameof(jobContext));
ArgUtil.NotNull(message, nameof(message));
// Create a new timeline record for 'Set up job'
IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), "Set up job", $"{nameof(JobExtension)}_Init", null, null);
List <IStep> preJobSteps = new List <IStep>();
List <IStep> jobSteps = new List <IStep>();
using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
{
try
{
context.Start();
context.Debug($"Starting: Set up job");
context.Output($"Current runner version: '{BuildConstants.RunnerPackage.Version}'");
var repoFullName = context.GetGitHubContext("repository");
ArgUtil.NotNull(repoFullName, nameof(repoFullName));
context.Debug($"Primary repository: {repoFullName}");
// Print proxy setting information for better diagnostic experience
if (!string.IsNullOrEmpty(HostContext.WebProxy.HttpProxyAddress))
{
context.Output($"Runner is running behind proxy server '{HostContext.WebProxy.HttpProxyAddress}' for all HTTP requests.");
}
if (!string.IsNullOrEmpty(HostContext.WebProxy.HttpsProxyAddress))
{
context.Output($"Runner is running behind proxy server '{HostContext.WebProxy.HttpsProxyAddress}' for all HTTPS requests.");
}
// Prepare the workflow directory
context.Output("Prepare workflow directory");
var directoryManager = HostContext.GetService <IPipelineDirectoryManager>();
TrackingConfig trackingConfig = directoryManager.PrepareDirectory(
context,
message.Workspace);
// Set the directory variables
context.Debug("Update context data");
string _workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
context.SetRunnerContext("workspace", Path.Combine(_workDirectory, trackingConfig.PipelineDirectory));
context.SetGitHubContext("workspace", Path.Combine(_workDirectory, trackingConfig.WorkspaceDirectory));
// Evaluate the job-level environment variables
context.Debug("Evaluating job-level environment variables");
var templateTrace = context.ToTemplateTraceWriter();
var schema = new PipelineTemplateSchemaFactory().CreateSchema();
var templateEvaluator = new PipelineTemplateEvaluator(templateTrace, schema);
foreach (var token in message.EnvironmentVariables)
{
var environmentVariables = templateEvaluator.EvaluateStepEnvironment(token, jobContext.ExpressionValues, VarUtil.EnvironmentVariableKeyComparer);
foreach (var pair in environmentVariables)
{
context.EnvironmentVariables[pair.Key] = pair.Value ?? string.Empty;
context.SetEnvContext(pair.Key, pair.Value ?? string.Empty);
}
}
// Evaluate the job container
context.Debug("Evaluating job container");
var container = templateEvaluator.EvaluateJobContainer(message.JobContainer, jobContext.ExpressionValues);
if (container != null)
{
jobContext.Container = new Container.ContainerInfo(HostContext, container);
}
// Evaluate the job service containers
context.Debug("Evaluating job service containers");
var serviceContainers = templateEvaluator.EvaluateJobServiceContainers(message.JobServiceContainers, jobContext.ExpressionValues);
if (serviceContainers?.Count > 0)
{
foreach (var pair in serviceContainers)
{
var networkAlias = pair.Key;
var serviceContainer = pair.Value;
jobContext.ServiceContainers.Add(new Container.ContainerInfo(HostContext, serviceContainer, false, networkAlias));
}
}
// Build up 2 lists of steps, pre-job, job
// Download actions not already in the cache
Trace.Info("Downloading actions");
var actionManager = HostContext.GetService <IActionManager>();
var prepareSteps = await actionManager.PrepareActionsAsync(context, message.Steps);
preJobSteps.AddRange(prepareSteps);
// Add start-container steps, record and stop-container steps
if (jobContext.Container != null || jobContext.ServiceContainers.Count > 0)
{
var containerProvider = HostContext.GetService <IContainerOperationProvider>();
var containers = new List <Container.ContainerInfo>();
if (jobContext.Container != null)
{
containers.Add(jobContext.Container);
}
containers.AddRange(jobContext.ServiceContainers);
preJobSteps.Add(new JobExtensionRunner(runAsync: containerProvider.StartContainersAsync,
condition: $"{PipelineTemplateConstants.Success}()",
displayName: "Initialize containers",
data: (object)containers));
}
// Add action steps
foreach (var step in message.Steps)
{
if (step.Type == Pipelines.StepType.Action)
{
var action = step as Pipelines.ActionStep;
Trace.Info($"Adding {action.DisplayName}.");
var actionRunner = HostContext.CreateService <IActionRunner>();
actionRunner.Action = action;
actionRunner.Stage = ActionRunStage.Main;
actionRunner.Condition = step.Condition;
var contextData = new Pipelines.ContextData.DictionaryContextData();
if (message.ContextData?.Count > 0)
{
foreach (var pair in message.ContextData)
{
contextData[pair.Key] = pair.Value;
}
}
actionRunner.TryEvaluateDisplayName(contextData, context);
jobSteps.Add(actionRunner);
}
}
// Create execution context for pre-job steps
foreach (var step in preJobSteps)
{
if (step is JobExtensionRunner)
{
JobExtensionRunner extensionStep = step as JobExtensionRunner;
ArgUtil.NotNull(extensionStep, extensionStep.DisplayName);
Guid stepId = Guid.NewGuid();
extensionStep.ExecutionContext = jobContext.CreateChild(stepId, extensionStep.DisplayName, null, null, stepId.ToString("N"));
}
}
// Create execution context for job steps
foreach (var step in jobSteps)
{
if (step is IActionRunner actionStep)
{
ArgUtil.NotNull(actionStep, step.DisplayName);
actionStep.ExecutionContext = jobContext.CreateChild(actionStep.Action.Id, actionStep.DisplayName, actionStep.Action.Name, actionStep.Action.ScopeName, actionStep.Action.ContextName);
}
}
List <IStep> steps = new List <IStep>();
steps.AddRange(preJobSteps);
steps.AddRange(jobSteps);
// Prepare for orphan process cleanup
_processCleanup = jobContext.Variables.GetBoolean("process.clean") ?? true;
if (_processCleanup)
{
// Set the RUNNER_TRACKING_ID env variable.
Environment.SetEnvironmentVariable(Constants.ProcessTrackingId, _processLookupId);
context.Debug("Collect running processes for tracking orphan processes.");
// Take a snapshot of current running processes
Dictionary <int, Process> processes = SnapshotProcesses();
foreach (var proc in processes)
{
// Pid_ProcessName
_existingProcesses.Add($"{proc.Key}_{proc.Value.ProcessName}");
}
}
return(steps);
}
catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
{
// Log the exception and cancel the JobExtension Initialization.
Trace.Error($"Caught cancellation exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Canceled;
throw;
}
catch (Exception ex)
{
// Log the error and fail the JobExtension Initialization.
Trace.Error($"Caught exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Failed;
throw;
}
finally
{
context.Debug("Finishing: Set up job");
context.Complete();
}
}
}
