// Download all required actions.
// Make sure all condition inputs are valid.
// Build up three list of steps for jobrunner (pre-job, job, post-job).
public async Task <List <IStep> > InitializeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message)
{
Trace.Entering();
ArgUtil.NotNull(jobContext, nameof(jobContext));
ArgUtil.NotNull(message, nameof(message));
// Create a new timeline record for 'Set up job'
IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), "Set up job", $"{nameof(JobExtension)}_Init", null, null);
List <IStep> preJobSteps = new List <IStep>();
List <IStep> jobSteps = new List <IStep>();
using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
{
try
{
context.Start();
context.Debug($"Starting: Set up job");
context.Output($"Current runner version: '{BuildConstants.RunnerPackage.Version}'");
var setting = HostContext.GetService <IConfigurationStore>().GetSettings();
var credFile = HostContext.GetConfigFile(WellKnownConfigFile.Credentials);
if (File.Exists(credFile))
{
var credData = IOUtil.LoadObject <CredentialData>(credFile);
if (credData != null &&
credData.Data.TryGetValue("clientId", out var clientId))
{
// print out HostName for self-hosted runner
context.Output($"Runner name: '{setting.AgentName}'");
if (message.Variables.TryGetValue("system.runnerGroupName", out VariableValue runnerGroupName))
{
context.Output($"Runner group name: '{runnerGroupName.Value}'");
}
context.Output($"Machine name: '{Environment.MachineName}'");
}
}
var setupInfoFile = HostContext.GetConfigFile(WellKnownConfigFile.SetupInfo);
if (File.Exists(setupInfoFile))
{
Trace.Info($"Load machine setup info from {setupInfoFile}");
try
{
var setupInfo = IOUtil.LoadObject <List <SetupInfo> >(setupInfoFile);
if (setupInfo?.Count > 0)
{
foreach (var info in setupInfo)
{
if (!string.IsNullOrEmpty(info?.Detail))
{
var groupName = info.Group;
if (string.IsNullOrEmpty(groupName))
{
groupName = "Machine Setup Info";
}
context.Output($"##[group]{groupName}");
var multiLines = info.Detail.Replace("\r\n", "\n").TrimEnd('\n').Split('\n');
foreach (var line in multiLines)
{
context.Output(line);
}
context.Output("##[endgroup]");
}
}
}
}
catch (Exception ex)
{
context.Output($"Fail to load and print machine setup info: {ex.Message}");
Trace.Error(ex);
}
}
try
{
var tokenPermissions = jobContext.Global.Variables.Get("system.github.token.permissions") ?? "";
if (!string.IsNullOrEmpty(tokenPermissions))
{
context.Output($"##[group]GITHUB_TOKEN Permissions");
var permissions = StringUtil.ConvertFromJson <Dictionary <string, string> >(tokenPermissions);
foreach (KeyValuePair <string, string> entry in permissions)
{
context.Output($"{entry.Key}: {entry.Value}");
}
context.Output("##[endgroup]");
}
}
catch (Exception ex)
{
context.Output($"Fail to parse and display GITHUB_TOKEN permissions list: {ex.Message}");
Trace.Error(ex);
}
var repoFullName = context.GetGitHubContext("repository");
ArgUtil.NotNull(repoFullName, nameof(repoFullName));
context.Debug($"Primary repository: {repoFullName}");
// Print proxy setting information for better diagnostic experience
if (!string.IsNullOrEmpty(HostContext.WebProxy.HttpProxyAddress))
{
context.Output($"Runner is running behind proxy server '{HostContext.WebProxy.HttpProxyAddress}' for all HTTP requests.");
}
if (!string.IsNullOrEmpty(HostContext.WebProxy.HttpsProxyAddress))
{
context.Output($"Runner is running behind proxy server '{HostContext.WebProxy.HttpsProxyAddress}' for all HTTPS requests.");
}
// Prepare the workflow directory
context.Output("Prepare workflow directory");
var directoryManager = HostContext.GetService <IPipelineDirectoryManager>();
TrackingConfig trackingConfig = directoryManager.PrepareDirectory(
context,
message.Workspace);
// Set the directory variables
context.Debug("Update context data");
string _workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
context.SetRunnerContext("workspace", Path.Combine(_workDirectory, trackingConfig.PipelineDirectory));
context.SetGitHubContext("workspace", Path.Combine(_workDirectory, trackingConfig.WorkspaceDirectory));
// Temporary hack for GHES alpha
var configurationStore = HostContext.GetService <IConfigurationStore>();
var runnerSettings = configurationStore.GetSettings();
if (string.IsNullOrEmpty(context.GetGitHubContext("server_url")) && !runnerSettings.IsHostedServer && !string.IsNullOrEmpty(runnerSettings.GitHubUrl))
{
var url = new Uri(runnerSettings.GitHubUrl);
var portInfo = url.IsDefaultPort ? string.Empty : $":{url.Port.ToString(CultureInfo.InvariantCulture)}";
context.SetGitHubContext("server_url", $"{url.Scheme}://{url.Host}{portInfo}");
context.SetGitHubContext("api_url", $"{url.Scheme}://{url.Host}{portInfo}/api/v3");
context.SetGitHubContext("graphql_url", $"{url.Scheme}://{url.Host}{portInfo}/api/graphql");
}
// Evaluate the job-level environment variables
context.Debug("Evaluating job-level environment variables");
var templateEvaluator = context.ToPipelineTemplateEvaluator();
foreach (var token in message.EnvironmentVariables)
{
var environmentVariables = templateEvaluator.EvaluateStepEnvironment(token, jobContext.ExpressionValues, jobContext.ExpressionFunctions, VarUtil.EnvironmentVariableKeyComparer);
foreach (var pair in environmentVariables)
{
context.Global.EnvironmentVariables[pair.Key] = pair.Value ?? string.Empty;
context.SetEnvContext(pair.Key, pair.Value ?? string.Empty);
}
}
// Evaluate the job container
context.Debug("Evaluating job container");
var container = templateEvaluator.EvaluateJobContainer(message.JobContainer, jobContext.ExpressionValues, jobContext.ExpressionFunctions);
if (container != null)
{
jobContext.Global.Container = new Container.ContainerInfo(HostContext, container);
}
// Evaluate the job service containers
context.Debug("Evaluating job service containers");
var serviceContainers = templateEvaluator.EvaluateJobServiceContainers(message.JobServiceContainers, jobContext.ExpressionValues, jobContext.ExpressionFunctions);
if (serviceContainers?.Count > 0)
{
foreach (var pair in serviceContainers)
{
var networkAlias = pair.Key;
var serviceContainer = pair.Value;
jobContext.Global.ServiceContainers.Add(new Container.ContainerInfo(HostContext, serviceContainer, false, networkAlias));
}
}
// Evaluate the job defaults
context.Debug("Evaluating job defaults");
foreach (var token in message.Defaults)
{
var defaults = token.AssertMapping("defaults");
if (defaults.Any(x => string.Equals(x.Key.AssertString("defaults key").Value, "run", StringComparison.OrdinalIgnoreCase)))
{
context.Global.JobDefaults["run"] = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
var defaultsRun = defaults.First(x => string.Equals(x.Key.AssertString("defaults key").Value, "run", StringComparison.OrdinalIgnoreCase));
var jobDefaults = templateEvaluator.EvaluateJobDefaultsRun(defaultsRun.Value, jobContext.ExpressionValues, jobContext.ExpressionFunctions);
foreach (var pair in jobDefaults)
{
if (!string.IsNullOrEmpty(pair.Value))
{
context.Global.JobDefaults["run"][pair.Key] = pair.Value;
}
}
}
}
// Build up 2 lists of steps, pre-job, job
// Download actions not already in the cache
Trace.Info("Downloading actions");
var actionManager = HostContext.GetService <IActionManager>();
var prepareResult = await actionManager.PrepareActionsAsync(context, message.Steps);
preJobSteps.AddRange(prepareResult.ContainerSetupSteps);
// Add start-container steps, record and stop-container steps
if (jobContext.Global.Container != null || jobContext.Global.ServiceContainers.Count > 0)
{
var containerProvider = HostContext.GetService <IContainerOperationProvider>();
var containers = new List <Container.ContainerInfo>();
if (jobContext.Global.Container != null)
{
containers.Add(jobContext.Global.Container);
}
containers.AddRange(jobContext.Global.ServiceContainers);
preJobSteps.Add(new JobExtensionRunner(runAsync: containerProvider.StartContainersAsync,
condition: $"{PipelineTemplateConstants.Success}()",
displayName: "Initialize containers",
data: (object)containers));
}
// Add action steps
foreach (var step in message.Steps)
{
if (step.Type == Pipelines.StepType.Action)
{
var action = step as Pipelines.ActionStep;
Trace.Info($"Adding {action.DisplayName}.");
var actionRunner = HostContext.CreateService <IActionRunner>();
actionRunner.Action = action;
actionRunner.Stage = ActionRunStage.Main;
actionRunner.Condition = step.Condition;
var contextData = new Pipelines.ContextData.DictionaryContextData();
if (message.ContextData?.Count > 0)
{
foreach (var pair in message.ContextData)
{
contextData[pair.Key] = pair.Value;
}
}
actionRunner.TryEvaluateDisplayName(contextData, context);
jobSteps.Add(actionRunner);
if (prepareResult.PreStepTracker.TryGetValue(step.Id, out var preStep))
{
Trace.Info($"Adding pre-{action.DisplayName}.");
preStep.TryEvaluateDisplayName(contextData, context);
preStep.DisplayName = $"Pre {preStep.DisplayName}";
preJobSteps.Add(preStep);
}
}
}
var intraActionStates = new Dictionary <Guid, Dictionary <string, string> >();
foreach (var preStep in prepareResult.PreStepTracker)
{
intraActionStates[preStep.Key] = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
}
// Create execution context for pre-job steps
foreach (var step in preJobSteps)
{
if (step is JobExtensionRunner)
{
JobExtensionRunner extensionStep = step as JobExtensionRunner;
ArgUtil.NotNull(extensionStep, extensionStep.DisplayName);
Guid stepId = Guid.NewGuid();
extensionStep.ExecutionContext = jobContext.CreateChild(stepId, extensionStep.DisplayName, null, null, stepId.ToString("N"));
}
else if (step is IActionRunner actionStep)
{
ArgUtil.NotNull(actionStep, step.DisplayName);
Guid stepId = Guid.NewGuid();
actionStep.ExecutionContext = jobContext.CreateChild(stepId, actionStep.DisplayName, stepId.ToString("N"), null, null, intraActionStates[actionStep.Action.Id]);
}
}
// Create execution context for job steps
foreach (var step in jobSteps)
{
if (step is IActionRunner actionStep)
{
ArgUtil.NotNull(actionStep, step.DisplayName);
intraActionStates.TryGetValue(actionStep.Action.Id, out var intraActionState);
actionStep.ExecutionContext = jobContext.CreateChild(actionStep.Action.Id, actionStep.DisplayName, actionStep.Action.Name, null, actionStep.Action.ContextName, intraActionState);
}
}
List <IStep> steps = new List <IStep>();
steps.AddRange(preJobSteps);
steps.AddRange(jobSteps);
// Prepare for orphan process cleanup
_processCleanup = jobContext.Global.Variables.GetBoolean("process.clean") ?? true;
if (_processCleanup)
{
// Set the RUNNER_TRACKING_ID env variable.
Environment.SetEnvironmentVariable(Constants.ProcessTrackingId, _processLookupId);
context.Debug("Collect running processes for tracking orphan processes.");
// Take a snapshot of current running processes
Dictionary <int, Process> processes = SnapshotProcesses();
foreach (var proc in processes)
{
// Pid_ProcessName
_existingProcesses.Add($"{proc.Key}_{proc.Value.ProcessName}");
}
}
jobContext.Global.EnvironmentVariables.TryGetValue(Constants.Runner.Features.DiskSpaceWarning, out var enableWarning);
if (StringUtil.ConvertToBoolean(enableWarning, defaultValue: true))
{
_diskSpaceCheckTask = CheckDiskSpaceAsync(context, _diskSpaceCheckToken.Token);
}
return(steps);
}
catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
{
// Log the exception and cancel the JobExtension Initialization.
Trace.Error($"Caught cancellation exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Canceled;
throw;
}
catch (FailedToResolveActionDownloadInfoException ex)
{
// Log the error and fail the JobExtension Initialization.
Trace.Error($"Caught exception from JobExtenion Initialization: {ex}");
context.InfrastructureError(ex.Message);
context.Result = TaskResult.Failed;
throw;
}
catch (Exception ex)
{
// Log the error and fail the JobExtension Initialization.
Trace.Error($"Caught exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Failed;
throw;
}
finally
{
context.Debug("Finishing: Set up job");
context.Complete();
}
}
}
// download all required tasks.
// make sure all task's condition inputs are valid.
// build up three list of steps for jobrunner. (pre-job, job, post-job)
public async Task <JobInitializeResult> InitializeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message)
{
Trace.Entering();
ArgUtil.NotNull(jobContext, nameof(jobContext));
ArgUtil.NotNull(message, nameof(message));
// create a new timeline record node for 'Initialize job'
IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("InitializeJob"), nameof(JobExtension));
JobInitializeResult initResult = new JobInitializeResult();
using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
{
try
{
context.Start();
context.Section(StringUtil.Loc("StepStarting", StringUtil.Loc("InitializeJob")));
// Set agent version variable.
context.Variables.Set(Constants.Variables.Agent.Version, Constants.Agent.Version);
context.Output(StringUtil.Loc("AgentVersion", Constants.Agent.Version));
// Print proxy setting information for better diagnostic experience
var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress))
{
context.Output(StringUtil.Loc("AgentRunningBehindProxy", agentWebProxy.ProxyAddress));
}
// Give job extension a chance to initialize
Trace.Info($"Run initial step from extension {this.GetType().Name}.");
InitializeJobExtension(context, message.Steps, message.Workspace);
// Download tasks if not already in the cache
Trace.Info("Downloading task definitions.");
var taskManager = HostContext.GetService <ITaskManager>();
await taskManager.DownloadAsync(context, message.Steps);
// Parse all Task conditions.
Trace.Info("Parsing all task's condition inputs.");
var expression = HostContext.GetService <IExpressionManager>();
Dictionary <Guid, IExpressionNode> taskConditionMap = new Dictionary <Guid, IExpressionNode>();
foreach (var task in message.Steps.OfType <Pipelines.TaskStep>())
{
IExpressionNode condition;
if (!string.IsNullOrEmpty(task.Condition))
{
context.Debug($"Task '{task.DisplayName}' has following condition: '{task.Condition}'.");
condition = expression.Parse(context, task.Condition);
}
else
{
condition = ExpressionManager.Succeeded;
}
taskConditionMap[task.Id] = condition;
}
#if OS_WINDOWS
// This is for internal testing and is not publicly supported. This will be removed from the agent at a later time.
var prepareScript = Environment.GetEnvironmentVariable("VSTS_AGENT_INIT_INTERNAL_TEMP_HACK");
ServiceEndpoint systemConnection = context.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
if (!string.IsNullOrEmpty(prepareScript) && context.Container == null)
{
var prepareStep = new ManagementScriptStep(
scriptPath: prepareScript,
condition: ExpressionManager.Succeeded,
displayName: "Agent Initialization");
Trace.Verbose($"Adding agent init script step.");
prepareStep.Initialize(HostContext);
prepareStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), prepareStep.DisplayName, nameof(ManagementScriptStep));
prepareStep.AccessToken = systemConnection.Authorization.Parameters["AccessToken"];
prepareStep.Condition = ExpressionManager.Succeeded;
initResult.PreJobSteps.Add(prepareStep);
}
#endif
// build up 3 lists of steps, pre-job, job, post-job
Stack <IStep> postJobStepsBuilder = new Stack <IStep>();
Dictionary <Guid, Variables> taskVariablesMapping = new Dictionary <Guid, Variables>();
if (context.Container != null || context.SidecarContainers.Count > 0)
{
var containerProvider = HostContext.GetService <IContainerOperationProvider>();
var containers = new List <Container.ContainerInfo>();
if (context.Container != null)
{
containers.Add(context.Container);
}
containers.AddRange(context.SidecarContainers);
initResult.PreJobSteps.Add(new JobExtensionRunner(runAsync: containerProvider.StartContainersAsync,
condition: ExpressionManager.Succeeded,
displayName: StringUtil.Loc("InitializeContainer"),
data: (object)containers));
postJobStepsBuilder.Push(new JobExtensionRunner(runAsync: containerProvider.StopContainersAsync,
condition: ExpressionManager.Always,
displayName: StringUtil.Loc("StopContainer"),
data: (object)containers));
}
foreach (var task in message.Steps.OfType <Pipelines.TaskStep>())
{
var taskDefinition = taskManager.Load(task);
List <string> warnings;
taskVariablesMapping[task.Id] = new Variables(HostContext, new Dictionary <string, VariableValue>(), out warnings);
// Add pre-job steps from Tasks
if (taskDefinition.Data?.PreJobExecution != null)
{
Trace.Info($"Adding Pre-Job {task.DisplayName}.");
var taskRunner = HostContext.CreateService <ITaskRunner>();
taskRunner.Task = task;
taskRunner.Stage = JobRunStage.PreJob;
taskRunner.Condition = taskConditionMap[task.Id];
initResult.PreJobSteps.Add(taskRunner);
}
// Add execution steps from Tasks
if (taskDefinition.Data?.Execution != null)
{
Trace.Verbose($"Adding {task.DisplayName}.");
var taskRunner = HostContext.CreateService <ITaskRunner>();
taskRunner.Task = task;
taskRunner.Stage = JobRunStage.Main;
taskRunner.Condition = taskConditionMap[task.Id];
initResult.JobSteps.Add(taskRunner);
}
// Add post-job steps from Tasks
if (taskDefinition.Data?.PostJobExecution != null)
{
Trace.Verbose($"Adding Post-Job {task.DisplayName}.");
var taskRunner = HostContext.CreateService <ITaskRunner>();
taskRunner.Task = task;
taskRunner.Stage = JobRunStage.PostJob;
taskRunner.Condition = ExpressionManager.Always;
postJobStepsBuilder.Push(taskRunner);
}
}
// Add pre-job step from Extension
Trace.Info("Adding pre-job step from extension.");
var extensionPreJobStep = GetExtensionPreJobStep(jobContext);
if (extensionPreJobStep != null)
{
initResult.PreJobSteps.Add(extensionPreJobStep);
}
// Add post-job step from Extension
Trace.Info("Adding post-job step from extension.");
var extensionPostJobStep = GetExtensionPostJobStep(jobContext);
if (extensionPostJobStep != null)
{
postJobStepsBuilder.Push(extensionPostJobStep);
}
// create execution context for all pre-job steps
foreach (var step in initResult.PreJobSteps)
{
#if OS_WINDOWS
if (step is ManagementScriptStep)
{
continue;
}
#endif
if (step is JobExtensionRunner)
{
JobExtensionRunner extensionStep = step as JobExtensionRunner;
ArgUtil.NotNull(extensionStep, extensionStep.DisplayName);
Guid stepId = Guid.NewGuid();
extensionStep.ExecutionContext = jobContext.CreateChild(stepId, extensionStep.DisplayName, stepId.ToString("N"));
}
else if (step is ITaskRunner)
{
ITaskRunner taskStep = step as ITaskRunner;
ArgUtil.NotNull(taskStep, step.DisplayName);
taskStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("PreJob", taskStep.DisplayName), taskStep.Task.Name, taskVariablesMapping[taskStep.Task.Id]);
}
}
// create task execution context for all job steps from task
foreach (var step in initResult.JobSteps)
{
ITaskRunner taskStep = step as ITaskRunner;
ArgUtil.NotNull(taskStep, step.DisplayName);
taskStep.ExecutionContext = jobContext.CreateChild(taskStep.Task.Id, taskStep.DisplayName, taskStep.Task.Name, taskVariablesMapping[taskStep.Task.Id]);
}
// Add post-job steps from Tasks
Trace.Info("Adding post-job steps from tasks.");
while (postJobStepsBuilder.Count > 0)
{
initResult.PostJobStep.Add(postJobStepsBuilder.Pop());
}
// create task execution context for all post-job steps from task
foreach (var step in initResult.PostJobStep)
{
if (step is JobExtensionRunner)
{
JobExtensionRunner extensionStep = step as JobExtensionRunner;
ArgUtil.NotNull(extensionStep, extensionStep.DisplayName);
Guid stepId = Guid.NewGuid();
extensionStep.ExecutionContext = jobContext.CreateChild(stepId, extensionStep.DisplayName, stepId.ToString("N"));
}
else if (step is ITaskRunner)
{
ITaskRunner taskStep = step as ITaskRunner;
ArgUtil.NotNull(taskStep, step.DisplayName);
taskStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("PostJob", taskStep.DisplayName), taskStep.Task.Name, taskVariablesMapping[taskStep.Task.Id]);
}
}
#if OS_WINDOWS
// Add script post steps.
// This is for internal testing and is not publicly supported. This will be removed from the agent at a later time.
var finallyScript = Environment.GetEnvironmentVariable("VSTS_AGENT_CLEANUP_INTERNAL_TEMP_HACK");
if (!string.IsNullOrEmpty(finallyScript) && context.Container == null)
{
var finallyStep = new ManagementScriptStep(
scriptPath: finallyScript,
condition: ExpressionManager.Always,
displayName: "Agent Cleanup");
Trace.Verbose($"Adding agent cleanup script step.");
finallyStep.Initialize(HostContext);
finallyStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), finallyStep.DisplayName, nameof(ManagementScriptStep));
finallyStep.Condition = ExpressionManager.Always;
finallyStep.AccessToken = systemConnection.Authorization.Parameters["AccessToken"];
initResult.PostJobStep.Add(finallyStep);
}
#endif
return(initResult);
}
catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
{
// Log the exception and cancel the JobExtension Initialization.
Trace.Error($"Caught cancellation exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Canceled;
throw;
}
catch (Exception ex)
{
// Log the error and fail the JobExtension Initialization.
Trace.Error($"Caught exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Failed;
throw;
}
finally
{
context.Section(StringUtil.Loc("StepFinishing", StringUtil.Loc("InitializeJob")));
context.Complete();
}
}
}
public void FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, DateTime jobStartTimeUtc)
{
Trace.Entering();
ArgUtil.NotNull(jobContext, nameof(jobContext));
// create a new timeline record node for 'Finalize job'
IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), "Complete job", $"{nameof(JobExtension)}_Final", null, null);
using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
{
try
{
context.Start();
context.Debug("Starting: Complete job");
Trace.Info("Initialize Env context");
#if OS_WINDOWS
var envContext = new DictionaryContextData();
#else
var envContext = new CaseSensitiveDictionaryContextData();
#endif
context.ExpressionValues["env"] = envContext;
foreach (var pair in context.Global.EnvironmentVariables)
{
envContext[pair.Key] = new StringContextData(pair.Value ?? string.Empty);
}
// Populate env context for each step
Trace.Info("Initialize steps context");
context.ExpressionValues["steps"] = context.Global.StepsContext.GetScope(context.ScopeName);
var templateEvaluator = context.ToPipelineTemplateEvaluator();
// Evaluate job outputs
if (message.JobOutputs != null && message.JobOutputs.Type != TokenType.Null)
{
try
{
context.Output($"Evaluate and set job outputs");
// Populate env context for each step
Trace.Info("Initialize Env context for evaluating job outputs");
var outputs = templateEvaluator.EvaluateJobOutput(message.JobOutputs, context.ExpressionValues, context.ExpressionFunctions);
foreach (var output in outputs)
{
if (string.IsNullOrEmpty(output.Value))
{
context.Debug($"Skip output '{output.Key}' since it's empty");
continue;
}
if (!string.Equals(output.Value, HostContext.SecretMasker.MaskSecrets(output.Value)))
{
context.Warning($"Skip output '{output.Key}' since it may contain secret.");
continue;
}
context.Output($"Set output '{output.Key}'");
jobContext.JobOutputs[output.Key] = output.Value;
}
}
catch (Exception ex)
{
context.Result = TaskResult.Failed;
context.Error($"Fail to evaluate job outputs");
context.Error(ex);
jobContext.Result = TaskResultUtil.MergeTaskResults(jobContext.Result, TaskResult.Failed);
}
}
// Evaluate environment data
if (jobContext.ActionsEnvironment?.Url != null && jobContext.ActionsEnvironment?.Url.Type != TokenType.Null)
{
try
{
context.Output($"Evaluate and set environment url");
var environmentUrlToken = templateEvaluator.EvaluateEnvironmentUrl(jobContext.ActionsEnvironment.Url, context.ExpressionValues, context.ExpressionFunctions);
var environmentUrl = environmentUrlToken.AssertString("environment.url");
if (!string.Equals(environmentUrl.Value, HostContext.SecretMasker.MaskSecrets(environmentUrl.Value)))
{
context.Warning($"Skip setting environment url as environment '{jobContext.ActionsEnvironment.Name}' may contain secret.");
}
else
{
context.Output($"Evaluated environment url: {environmentUrl}");
jobContext.ActionsEnvironment.Url = environmentUrlToken;
}
}
catch (Exception ex)
{
context.Result = TaskResult.Failed;
context.Error($"Failed to evaluate environment url");
context.Error(ex);
jobContext.Result = TaskResultUtil.MergeTaskResults(jobContext.Result, TaskResult.Failed);
}
}
if (context.Global.Variables.GetBoolean(Constants.Variables.Actions.RunnerDebug) ?? false)
{
Trace.Info("Support log upload starting.");
context.Output("Uploading runner diagnostic logs");
IDiagnosticLogManager diagnosticLogManager = HostContext.GetService <IDiagnosticLogManager>();
try
{
diagnosticLogManager.UploadDiagnosticLogs(executionContext: context, parentContext: jobContext, message: message, jobStartTimeUtc: jobStartTimeUtc);
Trace.Info("Support log upload complete.");
context.Output("Completed runner diagnostic log upload");
}
catch (Exception ex)
{
// Log the error but make sure we continue gracefully.
Trace.Info("Error uploading support logs.");
context.Output("Error uploading runner diagnostic logs");
Trace.Error(ex);
}
}
if (_processCleanup)
{
context.Output("Cleaning up orphan processes");
// Only check environment variable for any process that doesn't run before we invoke our process.
Dictionary <int, Process> currentProcesses = SnapshotProcesses();
foreach (var proc in currentProcesses)
{
if (proc.Key == Process.GetCurrentProcess().Id)
{
// skip for current process.
continue;
}
if (_existingProcesses.Contains($"{proc.Key}_{proc.Value.ProcessName}"))
{
Trace.Verbose($"Skip existing process. PID: {proc.Key} ({proc.Value.ProcessName})");
}
else
{
Trace.Info($"Inspecting process environment variables. PID: {proc.Key} ({proc.Value.ProcessName})");
string lookupId = null;
try
{
lookupId = proc.Value.GetEnvironmentVariable(HostContext, Constants.ProcessTrackingId);
}
catch (Exception ex)
{
Trace.Warning($"Ignore exception during read process environment variables: {ex.Message}");
Trace.Verbose(ex.ToString());
}
if (string.Equals(lookupId, _processLookupId, StringComparison.OrdinalIgnoreCase))
{
context.Output($"Terminate orphan process: pid ({proc.Key}) ({proc.Value.ProcessName})");
try
{
proc.Value.Kill();
}
catch (Exception ex)
{
Trace.Error("Catch exception during orphan process cleanup.");
Trace.Error(ex);
}
}
}
}
}
if (_diskSpaceCheckTask != null)
{
_diskSpaceCheckToken.Cancel();
}
}
catch (Exception ex)
{
// Log and ignore the error from JobExtension finalization.
Trace.Error($"Caught exception from JobExtension finalization: {ex}");
context.Output(ex.Message);
}
finally
{
context.Debug("Finishing: Complete job");
context.Complete();
}
}
}
// download all required tasks.
// make sure all task's condition inputs are valid.
// build up three list of steps for jobrunner. (pre-job, job, post-job)
public async Task <List <IStep> > InitializeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message)
{
Trace.Entering();
ArgUtil.NotNull(jobContext, nameof(jobContext));
ArgUtil.NotNull(message, nameof(message));
// create a new timeline record node for 'Initialize job'
IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("InitializeJob"), $"{nameof(JobExtension)}_Init");
List <IStep> preJobSteps = new List <IStep>();
List <IStep> jobSteps = new List <IStep>();
List <IStep> postJobSteps = new List <IStep>();
using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
{
try
{
context.Start();
context.Section(StringUtil.Loc("StepStarting", StringUtil.Loc("InitializeJob")));
// Set agent version variable.
context.SetVariable(Constants.Variables.Agent.Version, BuildConstants.AgentPackage.Version);
context.Output(StringUtil.Loc("AgentNameLog", context.Variables.Get(Constants.Variables.Agent.Name)));
context.Output(StringUtil.Loc("AgentMachineNameLog", context.Variables.Get(Constants.Variables.Agent.MachineName)));
context.Output(StringUtil.Loc("AgentVersion", BuildConstants.AgentPackage.Version));
// Machine specific setup info
OutputSetupInfo(context);
string imageVersion = System.Environment.GetEnvironmentVariable(Constants.ImageVersionVariable);
if (imageVersion != null)
{
context.Output(StringUtil.Loc("ImageVersionLog", imageVersion));
}
context.Output(StringUtil.Loc("UserNameLog", System.Environment.UserName));
// Print proxy setting information for better diagnostic experience
var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress))
{
context.Output(StringUtil.Loc("AgentRunningBehindProxy", agentWebProxy.ProxyAddress));
}
// Give job extension a chance to initialize
Trace.Info($"Run initial step from extension {this.GetType().Name}.");
InitializeJobExtension(context, message?.Steps, message?.Workspace);
// Download tasks if not already in the cache
Trace.Info("Downloading task definitions.");
var taskManager = HostContext.GetService <ITaskManager>();
await taskManager.DownloadAsync(context, message.Steps);
// Parse all Task conditions.
Trace.Info("Parsing all task's condition inputs.");
var expression = HostContext.GetService <IExpressionManager>();
Dictionary <Guid, IExpressionNode> taskConditionMap = new Dictionary <Guid, IExpressionNode>();
foreach (var task in message.Steps.OfType <Pipelines.TaskStep>())
{
IExpressionNode condition;
if (!string.IsNullOrEmpty(task.Condition))
{
context.Debug($"Task '{task.DisplayName}' has following condition: '{task.Condition}'.");
condition = expression.Parse(context, task.Condition);
}
else
{
condition = ExpressionManager.Succeeded;
}
task.DisplayName = context.Variables.ExpandValue(nameof(task.DisplayName), task.DisplayName);
taskConditionMap[task.Id] = condition;
}
context.Output("Checking job knob settings.");
foreach (var knob in Knob.GetAllKnobsFor <AgentKnobs>())
{
var value = knob.GetValue(jobContext);
if (value.Source.GetType() != typeof(BuiltInDefaultKnobSource))
{
var tag = "";
if (knob.IsDeprecated)
{
tag = "(DEPRECATED)";
}
else if (knob.IsExperimental)
{
tag = "(EXPERIMENTAL)";
}
var outputLine = $" Knob: {knob.Name} = {value.AsString()} Source: {value.Source.GetDisplayString()} {tag}";
if (knob.IsDeprecated)
{
context.Warning(outputLine);
}
else
{
context.Output(outputLine);
}
}
}
context.Output("Finished checking job knob settings.");
if (PlatformUtil.RunningOnWindows)
{
// This is for internal testing and is not publicly supported. This will be removed from the agent at a later time.
var prepareScript = Environment.GetEnvironmentVariable("VSTS_AGENT_INIT_INTERNAL_TEMP_HACK");
ServiceEndpoint systemConnection = context.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
if (!string.IsNullOrEmpty(prepareScript) && context.StepTarget() is HostInfo)
{
var prepareStep = new ManagementScriptStep(
scriptPath: prepareScript,
condition: ExpressionManager.Succeeded,
displayName: "Agent Initialization");
Trace.Verbose($"Adding agent init script step.");
prepareStep.Initialize(HostContext);
prepareStep.ExecutionContext = jobContext?.CreateChild(Guid.NewGuid(), prepareStep.DisplayName, nameof(ManagementScriptStep));
prepareStep.AccessToken = systemConnection.Authorization.Parameters["AccessToken"];
prepareStep.Condition = ExpressionManager.Succeeded;
preJobSteps.Add(prepareStep);
}
}
// build up 3 lists of steps, pre-job, job, post-job
Stack <IStep> postJobStepsBuilder = new Stack <IStep>();
Dictionary <Guid, Variables> taskVariablesMapping = new Dictionary <Guid, Variables>();
if (context.Containers.Count > 0 || context.SidecarContainers.Count > 0)
{
var containerProvider = HostContext.GetService <IContainerOperationProvider>();
var containers = new List <ContainerInfo>();
containers.AddRange(context.Containers);
containers.AddRange(context.SidecarContainers);
preJobSteps.Add(new JobExtensionRunner(runAsync: containerProvider.StartContainersAsync,
condition: ExpressionManager.Succeeded,
displayName: StringUtil.Loc("InitializeContainer"),
data: (object)containers));
postJobStepsBuilder.Push(new JobExtensionRunner(runAsync: containerProvider.StopContainersAsync,
condition: ExpressionManager.Always,
displayName: StringUtil.Loc("StopContainer"),
data: (object)containers));
}
Dictionary <Guid, List <TaskRestrictions> > taskRestrictionsMap = new Dictionary <Guid, List <TaskRestrictions> >();
foreach (var task in message?.Steps.OfType <Pipelines.TaskStep>())
{
var taskDefinition = taskManager.Load(task);
// Collect any task restrictions from the definition or step
var restrictions = new List <TaskRestrictions>();
if (taskDefinition.Data.Restrictions != null)
{
restrictions.Add(taskDefinition.Data.Restrictions);
}
if (string.Equals(WellKnownStepTargetStrings.Restricted, task.Target?.Commands, StringComparison.OrdinalIgnoreCase))
{
restrictions.Add(new TaskRestrictions()
{
Commands = new TaskCommandRestrictions()
{
Mode = TaskCommandMode.Restricted
}
});
}
if (task.Target?.SettableVariables != null)
{
restrictions.Add(new TaskRestrictions()
{
SettableVariables = task.Target.SettableVariables
});
}
taskRestrictionsMap[task.Id] = restrictions;
List <string> warnings;
taskVariablesMapping[task.Id] = new Variables(HostContext, new Dictionary <string, VariableValue>(), out warnings);
// Add pre-job steps from Tasks
if (taskDefinition.Data?.PreJobExecution != null)
{
Trace.Info($"Adding Pre-Job {task.DisplayName}.");
var taskRunner = HostContext.CreateService <ITaskRunner>();
taskRunner.Task = task;
taskRunner.Stage = JobRunStage.PreJob;
taskRunner.Condition = taskConditionMap[task.Id];
preJobSteps.Add(taskRunner);
}
// Add execution steps from Tasks
if (taskDefinition.Data?.Execution != null)
{
Trace.Verbose($"Adding {task.DisplayName}.");
var taskRunner = HostContext.CreateService <ITaskRunner>();
taskRunner.Task = task;
taskRunner.Stage = JobRunStage.Main;
taskRunner.Condition = taskConditionMap[task.Id];
jobSteps.Add(taskRunner);
}
// Add post-job steps from Tasks
if (taskDefinition.Data?.PostJobExecution != null)
{
Trace.Verbose($"Adding Post-Job {task.DisplayName}.");
var taskRunner = HostContext.CreateService <ITaskRunner>();
taskRunner.Task = task;
taskRunner.Stage = JobRunStage.PostJob;
taskRunner.Condition = ExpressionManager.Always;
postJobStepsBuilder.Push(taskRunner);
}
}
// Add pre-job step from Extension
Trace.Info("Adding pre-job step from extension.");
var extensionPreJobStep = GetExtensionPreJobStep(jobContext);
if (extensionPreJobStep != null)
{
preJobSteps.Add(extensionPreJobStep);
}
// Add post-job step from Extension
Trace.Info("Adding post-job step from extension.");
var extensionPostJobStep = GetExtensionPostJobStep(jobContext);
if (extensionPostJobStep != null)
{
postJobStepsBuilder.Push(extensionPostJobStep);
}
ArgUtil.NotNull(jobContext, nameof(jobContext)); // I am not sure why this is needed, but static analysis flagged all uses of jobContext below this point
// create execution context for all pre-job steps
foreach (var step in preJobSteps)
{
if (PlatformUtil.RunningOnWindows && step is ManagementScriptStep)
{
continue;
}
if (step is JobExtensionRunner)
{
JobExtensionRunner extensionStep = step as JobExtensionRunner;
ArgUtil.NotNull(extensionStep, extensionStep.DisplayName);
Guid stepId = Guid.NewGuid();
extensionStep.ExecutionContext = jobContext.CreateChild(stepId, extensionStep.DisplayName, stepId.ToString("N"));
}
else if (step is ITaskRunner)
{
ITaskRunner taskStep = step as ITaskRunner;
ArgUtil.NotNull(taskStep, step.DisplayName);
taskStep.ExecutionContext = jobContext.CreateChild(
Guid.NewGuid(),
StringUtil.Loc("PreJob", taskStep.DisplayName),
taskStep.Task.Name,
taskVariablesMapping[taskStep.Task.Id],
outputForward: true,
taskRestrictions: taskRestrictionsMap[taskStep.Task.Id]);
}
}
// create task execution context for all job steps from task
foreach (var step in jobSteps)
{
ITaskRunner taskStep = step as ITaskRunner;
ArgUtil.NotNull(taskStep, step.DisplayName);
taskStep.ExecutionContext = jobContext.CreateChild(
taskStep.Task.Id,
taskStep.DisplayName,
taskStep.Task.Name,
taskVariablesMapping[taskStep.Task.Id],
outputForward: true,
taskRestrictions: taskRestrictionsMap[taskStep.Task.Id]);
}
// Add post-job steps from Tasks
Trace.Info("Adding post-job steps from tasks.");
while (postJobStepsBuilder.Count > 0)
{
postJobSteps.Add(postJobStepsBuilder.Pop());
}
// create task execution context for all post-job steps from task
foreach (var step in postJobSteps)
{
if (step is JobExtensionRunner)
{
JobExtensionRunner extensionStep = step as JobExtensionRunner;
ArgUtil.NotNull(extensionStep, extensionStep.DisplayName);
Guid stepId = Guid.NewGuid();
extensionStep.ExecutionContext = jobContext.CreateChild(stepId, extensionStep.DisplayName, stepId.ToString("N"));
}
else if (step is ITaskRunner)
{
ITaskRunner taskStep = step as ITaskRunner;
ArgUtil.NotNull(taskStep, step.DisplayName);
taskStep.ExecutionContext = jobContext.CreateChild(
Guid.NewGuid(),
StringUtil.Loc("PostJob", taskStep.DisplayName),
taskStep.Task.Name,
taskVariablesMapping[taskStep.Task.Id],
outputForward: true,
taskRestrictions: taskRestrictionsMap[taskStep.Task.Id]);
}
}
if (PlatformUtil.RunningOnWindows)
{
// Add script post steps.
// This is for internal testing and is not publicly supported. This will be removed from the agent at a later time.
var finallyScript = Environment.GetEnvironmentVariable("VSTS_AGENT_CLEANUP_INTERNAL_TEMP_HACK");
if (!string.IsNullOrEmpty(finallyScript) && context.StepTarget() is HostInfo)
{
var finallyStep = new ManagementScriptStep(
scriptPath: finallyScript,
condition: ExpressionManager.Always,
displayName: "Agent Cleanup");
Trace.Verbose($"Adding agent cleanup script step.");
finallyStep.Initialize(HostContext);
finallyStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), finallyStep.DisplayName, nameof(ManagementScriptStep));
finallyStep.Condition = ExpressionManager.Always;
ServiceEndpoint systemConnection = context.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
finallyStep.AccessToken = systemConnection.Authorization.Parameters["AccessToken"];
postJobSteps.Add(finallyStep);
}
}
List <IStep> steps = new List <IStep>();
steps.AddRange(preJobSteps);
steps.AddRange(jobSteps);
steps.AddRange(postJobSteps);
// Start agent log plugin host process
var logPlugin = HostContext.GetService <IAgentLogPlugin>();
await logPlugin.StartAsync(context, steps, jobContext.CancellationToken);
// Prepare for orphan process cleanup
_processCleanup = jobContext.Variables.GetBoolean("process.clean") ?? true;
if (_processCleanup)
{
// Set the VSTS_PROCESS_LOOKUP_ID env variable.
context.SetVariable(Constants.ProcessLookupId, _processLookupId, false, false);
context.Output("Start tracking orphan processes.");
// Take a snapshot of current running processes
Dictionary <int, Process> processes = SnapshotProcesses();
foreach (var proc in processes)
{
// Pid_ProcessName
_existingProcesses.Add($"{proc.Key}_{proc.Value.ProcessName}");
}
}
_taskKeyCleanup = jobContext.Variables.GetBoolean("process.cleanTaskKey") ?? true;
return(steps);
}
catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
{
// Log the exception and cancel the JobExtension Initialization.
Trace.Error($"Caught cancellation exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Canceled;
throw;
}
catch (Exception ex)
{
// Log the error and fail the JobExtension Initialization.
Trace.Error($"Caught exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Failed;
throw;
}
finally
{
context.Section(StringUtil.Loc("StepFinishing", StringUtil.Loc("InitializeJob")));
context.Complete();
}
}
}
public async Task FinalizeJob(IExecutionContext jobContext)
{
Trace.Entering();
ArgUtil.NotNull(jobContext, nameof(jobContext));
// create a new timeline record node for 'Finalize job'
IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("FinalizeJob"), $"{nameof(JobExtension)}_Final");
using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
{
try
{
context.Start();
context.Section(StringUtil.Loc("StepStarting", StringUtil.Loc("FinalizeJob")));
// Wait for agent log plugin process exits
var logPlugin = HostContext.GetService <IAgentLogPlugin>();
try
{
await logPlugin.WaitAsync(context);
}
catch (Exception ex)
{
// Log and ignore the error from log plugin finalization.
Trace.Error($"Caught exception from log plugin finalization: {ex}");
context.Output(ex.Message);
}
if (_taskKeyCleanup)
{
context.Output("Cleaning up task key");
string taskKeyFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), ".taskkey");
if (File.Exists(taskKeyFile))
{
try
{
File.Delete(taskKeyFile);
}
catch (Exception ex)
{
Trace.Error($"Caught exception while attempting to delete taskKey file {taskKeyFile}: {ex}");
context.Output(ex.Message);
}
}
}
if (_processCleanup)
{
context.Output("Start cleaning up orphan processes.");
// Only check environment variable for any process that doesn't run before we invoke our process.
Dictionary <int, Process> currentProcesses = SnapshotProcesses();
foreach (var proc in currentProcesses)
{
if (_existingProcesses.Contains($"{proc.Key}_{proc.Value.ProcessName}"))
{
Trace.Verbose($"Skip existing process. PID: {proc.Key} ({proc.Value.ProcessName})");
}
else
{
Trace.Info($"Inspecting process environment variables. PID: {proc.Key} ({proc.Value.ProcessName})");
string lookupId = null;
try
{
lookupId = proc.Value.GetEnvironmentVariable(HostContext, Constants.ProcessLookupId);
}
catch (Exception ex)
{
Trace.Warning($"Ignore exception during read process environment variables: {ex.Message}");
Trace.Verbose(ex.ToString());
}
if (string.Equals(lookupId, _processLookupId, StringComparison.OrdinalIgnoreCase))
{
context.Output($"Terminate orphan process: pid ({proc.Key}) ({proc.Value.ProcessName})");
try
{
proc.Value.Kill();
}
catch (Exception ex)
{
Trace.Error("Catch exception during orphan process cleanup.");
Trace.Error(ex);
}
}
}
}
}
}
catch (Exception ex)
{
// Log and ignore the error from JobExtension finalization.
Trace.Error($"Caught exception from JobExtension finalization: {ex}");
context.Output(ex.Message);
}
finally
{
context.Section(StringUtil.Loc("StepFinishing", StringUtil.Loc("FinalizeJob")));
context.Complete();
}
}
}
// Download all required actions.
// Make sure all condition inputs are valid.
// Build up three list of steps for jobrunner (pre-job, job, post-job).
public async Task <List <IStep> > InitializeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message)
{
Trace.Entering();
ArgUtil.NotNull(jobContext, nameof(jobContext));
ArgUtil.NotNull(message, nameof(message));
// Create a new timeline record for 'Set up job'
IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), "Set up job", $"{nameof(JobExtension)}_Init", null, null);
List <IStep> preJobSteps = new List <IStep>();
List <IStep> jobSteps = new List <IStep>();
using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
{
try
{
context.Start();
context.Debug($"Starting: Set up job");
context.Output($"Current runner version: '{BuildConstants.RunnerPackage.Version}'");
var setupInfoFile = HostContext.GetConfigFile(WellKnownConfigFile.SetupInfo);
if (File.Exists(setupInfoFile))
{
Trace.Info($"Load machine setup info from {setupInfoFile}");
try
{
var setupInfo = IOUtil.LoadObject <List <SetupInfo> >(setupInfoFile);
if (setupInfo?.Count > 0)
{
foreach (var info in setupInfo)
{
if (!string.IsNullOrEmpty(info?.Detail))
{
var groupName = info.Group;
if (string.IsNullOrEmpty(groupName))
{
groupName = "Machine Setup Info";
}
context.Output($"##[group]{groupName}");
var multiLines = info.Detail.Replace("\r\n", "\n").TrimEnd('\n').Split('\n');
foreach (var line in multiLines)
{
context.Output(line);
}
context.Output("##[endgroup]");
}
}
}
}
catch (Exception ex)
{
context.Output($"Fail to load and print machine setup info: {ex.Message}");
Trace.Error(ex);
}
}
var repoFullName = context.GetGitHubContext("repository");
ArgUtil.NotNull(repoFullName, nameof(repoFullName));
context.Debug($"Primary repository: {repoFullName}");
// Print proxy setting information for better diagnostic experience
if (!string.IsNullOrEmpty(HostContext.WebProxy.HttpProxyAddress))
{
context.Output($"Runner is running behind proxy server '{HostContext.WebProxy.HttpProxyAddress}' for all HTTP requests.");
}
if (!string.IsNullOrEmpty(HostContext.WebProxy.HttpsProxyAddress))
{
context.Output($"Runner is running behind proxy server '{HostContext.WebProxy.HttpsProxyAddress}' for all HTTPS requests.");
}
// Prepare the workflow directory
context.Output("Prepare workflow directory");
var directoryManager = HostContext.GetService <IPipelineDirectoryManager>();
TrackingConfig trackingConfig = directoryManager.PrepareDirectory(
context,
message.Workspace);
// Set the directory variables
context.Debug("Update context data");
string _workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
context.SetRunnerContext("workspace", Path.Combine(_workDirectory, trackingConfig.PipelineDirectory));
context.SetGitHubContext("workspace", Path.Combine(_workDirectory, trackingConfig.WorkspaceDirectory));
// Evaluate the job-level environment variables
context.Debug("Evaluating job-level environment variables");
var templateEvaluator = context.ToPipelineTemplateEvaluator();
foreach (var token in message.EnvironmentVariables)
{
var environmentVariables = templateEvaluator.EvaluateStepEnvironment(token, jobContext.ExpressionValues, VarUtil.EnvironmentVariableKeyComparer);
foreach (var pair in environmentVariables)
{
context.EnvironmentVariables[pair.Key] = pair.Value ?? string.Empty;
context.SetEnvContext(pair.Key, pair.Value ?? string.Empty);
}
}
// Evaluate the job container
context.Debug("Evaluating job container");
var container = templateEvaluator.EvaluateJobContainer(message.JobContainer, jobContext.ExpressionValues);
if (container != null)
{
jobContext.Container = new Container.ContainerInfo(HostContext, container);
}
// Evaluate the job service containers
context.Debug("Evaluating job service containers");
var serviceContainers = templateEvaluator.EvaluateJobServiceContainers(message.JobServiceContainers, jobContext.ExpressionValues);
if (serviceContainers?.Count > 0)
{
foreach (var pair in serviceContainers)
{
var networkAlias = pair.Key;
var serviceContainer = pair.Value;
jobContext.ServiceContainers.Add(new Container.ContainerInfo(HostContext, serviceContainer, false, networkAlias));
}
}
// Evaluate the job defaults
context.Debug("Evaluating job defaults");
foreach (var token in message.Defaults)
{
var defaults = token.AssertMapping("defaults");
if (defaults.Any(x => string.Equals(x.Key.AssertString("defaults key").Value, "run", StringComparison.OrdinalIgnoreCase)))
{
context.JobDefaults["run"] = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
var defaultsRun = defaults.First(x => string.Equals(x.Key.AssertString("defaults key").Value, "run", StringComparison.OrdinalIgnoreCase));
var jobDefaults = templateEvaluator.EvaluateJobDefaultsRun(defaultsRun.Value, jobContext.ExpressionValues);
foreach (var pair in jobDefaults)
{
if (!string.IsNullOrEmpty(pair.Value))
{
context.JobDefaults["run"][pair.Key] = pair.Value;
}
}
}
}
// Build up 2 lists of steps, pre-job, job
// Download actions not already in the cache
Trace.Info("Downloading actions");
var actionManager = HostContext.GetService <IActionManager>();
var prepareSteps = await actionManager.PrepareActionsAsync(context, message.Steps);
preJobSteps.AddRange(prepareSteps);
// Add start-container steps, record and stop-container steps
if (jobContext.Container != null || jobContext.ServiceContainers.Count > 0)
{
var containerProvider = HostContext.GetService <IContainerOperationProvider>();
var containers = new List <Container.ContainerInfo>();
if (jobContext.Container != null)
{
containers.Add(jobContext.Container);
}
containers.AddRange(jobContext.ServiceContainers);
preJobSteps.Add(new JobExtensionRunner(runAsync: containerProvider.StartContainersAsync,
condition: $"{PipelineTemplateConstants.Success}()",
displayName: "Initialize containers",
data: (object)containers));
}
// Add action steps
foreach (var step in message.Steps)
{
if (step.Type == Pipelines.StepType.Action)
{
var action = step as Pipelines.ActionStep;
Trace.Info($"Adding {action.DisplayName}.");
var actionRunner = HostContext.CreateService <IActionRunner>();
actionRunner.Action = action;
actionRunner.Stage = ActionRunStage.Main;
actionRunner.Condition = step.Condition;
var contextData = new Pipelines.ContextData.DictionaryContextData();
if (message.ContextData?.Count > 0)
{
foreach (var pair in message.ContextData)
{
contextData[pair.Key] = pair.Value;
}
}
actionRunner.TryEvaluateDisplayName(contextData, context);
jobSteps.Add(actionRunner);
}
}
// Create execution context for pre-job steps
foreach (var step in preJobSteps)
{
if (step is JobExtensionRunner)
{
JobExtensionRunner extensionStep = step as JobExtensionRunner;
ArgUtil.NotNull(extensionStep, extensionStep.DisplayName);
Guid stepId = Guid.NewGuid();
extensionStep.ExecutionContext = jobContext.CreateChild(stepId, extensionStep.DisplayName, null, null, stepId.ToString("N"));
}
}
// Create execution context for job steps
foreach (var step in jobSteps)
{
if (step is IActionRunner actionStep)
{
ArgUtil.NotNull(actionStep, step.DisplayName);
actionStep.ExecutionContext = jobContext.CreateChild(actionStep.Action.Id, actionStep.DisplayName, actionStep.Action.Name, actionStep.Action.ScopeName, actionStep.Action.ContextName);
}
}
List <IStep> steps = new List <IStep>();
steps.AddRange(preJobSteps);
steps.AddRange(jobSteps);
// Prepare for orphan process cleanup
_processCleanup = jobContext.Variables.GetBoolean("process.clean") ?? true;
if (_processCleanup)
{
// Set the RUNNER_TRACKING_ID env variable.
Environment.SetEnvironmentVariable(Constants.ProcessTrackingId, _processLookupId);
context.Debug("Collect running processes for tracking orphan processes.");
// Take a snapshot of current running processes
Dictionary <int, Process> processes = SnapshotProcesses();
foreach (var proc in processes)
{
// Pid_ProcessName
_existingProcesses.Add($"{proc.Key}_{proc.Value.ProcessName}");
}
}
return(steps);
}
catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
{
// Log the exception and cancel the JobExtension Initialization.
Trace.Error($"Caught cancellation exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Canceled;
throw;
}
catch (Exception ex)
{
// Log the error and fail the JobExtension Initialization.
Trace.Error($"Caught exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Failed;
throw;
}
finally
{
context.Debug("Finishing: Set up job");
context.Complete();
}
}
}
// download all required tasks.
// make sure all task's condition inputs are valid.
// build up three list of steps for jobrunner. (pre-job, job, post-job)
public async Task <JobInitializeResult> InitializeJob(IExecutionContext jobContext, AgentJobRequestMessage message)
{
Trace.Entering();
ArgUtil.NotNull(jobContext, nameof(jobContext));
ArgUtil.NotNull(message, nameof(message));
// create a new timeline record node for 'Initialize job'
IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("InitializeJob"));
JobInitializeResult initResult = new JobInitializeResult();
using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
{
try
{
context.Start();
context.Section(StringUtil.Loc("StepStarting", StringUtil.Loc("InitializeJob")));
// Give job extension a chance to initalize
Trace.Info($"Run initial step from extension {this.GetType().Name}.");
InitializeJobExtension(context);
// Download tasks if not already in the cache
Trace.Info("Downloading task definitions.");
var taskManager = HostContext.GetService <ITaskManager>();
await taskManager.DownloadAsync(context, message.Tasks);
// Parse all Task conditions.
Trace.Info("Parsing all task's condition inputs.");
var expression = HostContext.GetService <IExpressionManager>();
Dictionary <Guid, INode> taskConditionMap = new Dictionary <Guid, INode>();
foreach (var task in message.Tasks)
{
INode condition;
if (!string.IsNullOrEmpty(task.Condition))
{
context.Debug($"Task '{task.DisplayName}' has following condition: '{task.Condition}'.");
condition = expression.Parse(context, task.Condition);
}
else if (task.AlwaysRun)
{
condition = ExpressionManager.SucceededOrFailed;
}
else
{
condition = ExpressionManager.Succeeded;
}
taskConditionMap[task.InstanceId] = condition;
}
#if OS_WINDOWS
// This is for internal testing and is not publicly supported. This will be removed from the agent at a later time.
var prepareScript = Environment.GetEnvironmentVariable("VSTS_AGENT_INIT_INTERNAL_TEMP_HACK");
if (!string.IsNullOrEmpty(prepareScript))
{
var prepareStep = new ManagementScriptStep(
scriptPath: prepareScript,
condition: ExpressionManager.Succeeded,
displayName: "Agent Initialization");
Trace.Verbose($"Adding agent init script step.");
prepareStep.Initialize(HostContext);
prepareStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), prepareStep.DisplayName);
prepareStep.AccessToken = message.Environment.SystemConnection.Authorization.Parameters["AccessToken"];
initResult.PreJobSteps.Add(prepareStep);
}
#endif
// build up 3 lists of steps, pre-job, job, post-job
Stack <IStep> postJobStepsBuilder = new Stack <IStep>();
Dictionary <Guid, Variables> taskVariablesMapping = new Dictionary <Guid, Variables>();
foreach (var taskInstance in message.Tasks)
{
var taskDefinition = taskManager.Load(taskInstance);
List <string> warnings;
taskVariablesMapping[taskInstance.InstanceId] = new Variables(HostContext, new Dictionary <string, string>(), message.Environment.MaskHints, out warnings);
// Add pre-job steps from Tasks
if (taskDefinition.Data?.PreJobExecution != null)
{
Trace.Info($"Adding Pre-Job {taskInstance.DisplayName}.");
var taskRunner = HostContext.CreateService <ITaskRunner>();
taskRunner.TaskInstance = taskInstance;
taskRunner.Stage = JobRunStage.PreJob;
taskRunner.Condition = taskConditionMap[taskInstance.InstanceId];
initResult.PreJobSteps.Add(taskRunner);
}
// Add execution steps from Tasks
if (taskDefinition.Data?.Execution != null)
{
Trace.Verbose($"Adding {taskInstance.DisplayName}.");
var taskRunner = HostContext.CreateService <ITaskRunner>();
taskRunner.TaskInstance = taskInstance;
taskRunner.Stage = JobRunStage.Main;
taskRunner.Condition = taskConditionMap[taskInstance.InstanceId];
initResult.JobSteps.Add(taskRunner);
}
// Add post-job steps from Tasks
if (taskDefinition.Data?.PostJobExecution != null)
{
Trace.Verbose($"Adding Post-Job {taskInstance.DisplayName}.");
var taskRunner = HostContext.CreateService <ITaskRunner>();
taskRunner.TaskInstance = taskInstance;
taskRunner.Stage = JobRunStage.PostJob;
taskRunner.Condition = taskConditionMap[taskInstance.InstanceId];
postJobStepsBuilder.Push(taskRunner);
}
}
// create task execution context for all pre-job steps from task
foreach (var step in initResult.PreJobSteps)
{
#if OS_WINDOWS
if (step is ManagementScriptStep)
{
continue;
}
#endif
ITaskRunner taskStep = step as ITaskRunner;
ArgUtil.NotNull(taskStep, step.DisplayName);
taskStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("PreJob", taskStep.DisplayName), taskVariablesMapping[taskStep.TaskInstance.InstanceId]);
}
// Add pre-job step from Extension
Trace.Info("Adding pre-job step from extension.");
var extensionPreJobStep = GetExtensionPreJobStep(jobContext);
if (extensionPreJobStep != null)
{
initResult.PreJobSteps.Add(extensionPreJobStep);
}
// create task execution context for all job steps from task
foreach (var step in initResult.JobSteps)
{
ITaskRunner taskStep = step as ITaskRunner;
ArgUtil.NotNull(taskStep, step.DisplayName);
taskStep.ExecutionContext = jobContext.CreateChild(taskStep.TaskInstance.InstanceId, taskStep.DisplayName, taskVariablesMapping[taskStep.TaskInstance.InstanceId]);
}
// Add post-job steps from Tasks
Trace.Info("Adding post-job steps from tasks.");
while (postJobStepsBuilder.Count > 0)
{
initResult.PostJobStep.Add(postJobStepsBuilder.Pop());
}
// create task execution context for all post-job steps from task
foreach (var step in initResult.PostJobStep)
{
ITaskRunner taskStep = step as ITaskRunner;
ArgUtil.NotNull(taskStep, step.DisplayName);
taskStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("PostJob", taskStep.DisplayName), taskVariablesMapping[taskStep.TaskInstance.InstanceId]);
}
// Add post-job step from Extension
Trace.Info("Adding post-job step from extension.");
var extensionPostJobStep = GetExtensionPostJobStep(jobContext);
if (extensionPostJobStep != null)
{
initResult.PostJobStep.Add(extensionPostJobStep);
}
#if OS_WINDOWS
// Add script post steps.
// This is for internal testing and is not publicly supported. This will be removed from the agent at a later time.
var finallyScript = Environment.GetEnvironmentVariable("VSTS_AGENT_CLEANUP_INTERNAL_TEMP_HACK");
if (!string.IsNullOrEmpty(finallyScript))
{
var finallyStep = new ManagementScriptStep(
scriptPath: finallyScript,
condition: ExpressionManager.Always,
displayName: "Agent Cleanup");
Trace.Verbose($"Adding agent cleanup script step.");
finallyStep.Initialize(HostContext);
finallyStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), finallyStep.DisplayName);
finallyStep.AccessToken = message.Environment.SystemConnection.Authorization.Parameters["AccessToken"];
initResult.PostJobStep.Add(finallyStep);
}
#endif
return(initResult);
}
catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
{
// Log the exception and cancel the JobExtension Initialization.
Trace.Error($"Caught cancellation exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Canceled;
throw;
}
catch (Exception ex)
{
// Log the error and fail the JobExtension Initialization.
Trace.Error($"Caught exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Failed;
throw;
}
finally
{
context.Section(StringUtil.Loc("StepFinishing", StringUtil.Loc("InitializeJob")));
context.Complete();
}
}
}
public void FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, DateTime jobStartTimeUtc)
{
Trace.Entering();
ArgUtil.NotNull(jobContext, nameof(jobContext));
// create a new timeline record node for 'Finalize job'
IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), "Complete job", $"{nameof(JobExtension)}_Final", null, null);
using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
{
try
{
context.Start();
context.Debug("Starting: Complete job");
if (context.Variables.GetBoolean(Constants.Variables.Actions.RunnerDebug) ?? false)
{
Trace.Info("Support log upload starting.");
context.Output("Uploading runner diagnostic logs");
IDiagnosticLogManager diagnosticLogManager = HostContext.GetService <IDiagnosticLogManager>();
try
{
diagnosticLogManager.UploadDiagnosticLogs(executionContext: context, parentContext: jobContext, message: message, jobStartTimeUtc: jobStartTimeUtc);
Trace.Info("Support log upload complete.");
context.Output("Completed runner diagnostic log upload");
}
catch (Exception ex)
{
// Log the error but make sure we continue gracefully.
Trace.Info("Error uploading support logs.");
context.Output("Error uploading runner diagnostic logs");
Trace.Error(ex);
}
}
if (_processCleanup)
{
context.Output("Cleaning up orphan processes");
// Only check environment variable for any process that doesn't run before we invoke our process.
Dictionary <int, Process> currentProcesses = SnapshotProcesses();
foreach (var proc in currentProcesses)
{
if (proc.Key == Process.GetCurrentProcess().Id)
{
// skip for current process.
continue;
}
if (_existingProcesses.Contains($"{proc.Key}_{proc.Value.ProcessName}"))
{
Trace.Verbose($"Skip existing process. PID: {proc.Key} ({proc.Value.ProcessName})");
}
else
{
Trace.Info($"Inspecting process environment variables. PID: {proc.Key} ({proc.Value.ProcessName})");
string lookupId = null;
try
{
lookupId = proc.Value.GetEnvironmentVariable(HostContext, Constants.ProcessTrackingId);
}
catch (Exception ex)
{
Trace.Warning($"Ignore exception during read process environment variables: {ex.Message}");
Trace.Verbose(ex.ToString());
}
if (string.Equals(lookupId, _processLookupId, StringComparison.OrdinalIgnoreCase))
{
context.Output($"Terminate orphan process: pid ({proc.Key}) ({proc.Value.ProcessName})");
try
{
proc.Value.Kill();
}
catch (Exception ex)
{
Trace.Error("Catch exception during orphan process cleanup.");
Trace.Error(ex);
}
}
}
}
}
}
catch (Exception ex)
{
// Log and ignore the error from JobExtension finalization.
Trace.Error($"Caught exception from JobExtension finalization: {ex}");
context.Output(ex.Message);
}
finally
{
context.Debug("Finishing: Complete job");
context.Complete();
}
}
}