public void AddIssue_CountWarningsErrors()
{
using (TestHostContext hc = CreateTestContext())
{
// Arrange: Create a job request message.
TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
TimelineReference timeline = new TimelineReference();
Guid jobId = Guid.NewGuid();
string jobName = "some job name";
var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary <string, VariableValue>(), new List <MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List <Pipelines.ActionStep>(), null, null, null, null);
jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
{
Alias = Pipelines.PipelineConstants.SelfAlias,
Id = "github",
Version = "sha1"
});
jobRequest.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData();
// Arrange: Setup the paging logger.
var pagingLogger = new Mock <IPagingLogger>();
var jobServerQueue = new Mock <IJobServerQueue>();
jobServerQueue.Setup(x => x.QueueTimelineRecordUpdate(It.IsAny <Guid>(), It.IsAny <TimelineRecord>()));
hc.EnqueueInstance(pagingLogger.Object);
hc.SetSingleton(jobServerQueue.Object);
var ec = new Runner.Worker.ExecutionContext();
ec.Initialize(hc);
// Act.
ec.InitializeJob(jobRequest, CancellationToken.None);
ec.AddIssue(new Issue()
{
Type = IssueType.Error, Message = "error"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Error, Message = "error"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Error, Message = "error"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Error, Message = "error"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Error, Message = "error"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Error, Message = "error"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Error, Message = "error"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Error, Message = "error"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Error, Message = "error"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Error, Message = "error"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Error, Message = "error"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Error, Message = "error"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Error, Message = "error"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Error, Message = "error"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Error, Message = "error"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Warning, Message = "warning"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Warning, Message = "warning"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Warning, Message = "warning"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Warning, Message = "warning"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Warning, Message = "warning"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Warning, Message = "warning"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Warning, Message = "warning"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Warning, Message = "warning"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Warning, Message = "warning"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Warning, Message = "warning"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Warning, Message = "warning"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Warning, Message = "warning"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Warning, Message = "warning"
});
ec.AddIssue(new Issue()
{
Type = IssueType.Warning, Message = "warning"
});
ec.Complete();
// Assert.
jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny <Guid>(), It.Is <TimelineRecord>(t => t.ErrorCount == 15)), Times.AtLeastOnce);
jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny <Guid>(), It.Is <TimelineRecord>(t => t.WarningCount == 14)), Times.AtLeastOnce);
jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny <Guid>(), It.Is <TimelineRecord>(t => t.Issues.Where(i => i.Type == IssueType.Error).Count() == 10)), Times.AtLeastOnce);
jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny <Guid>(), It.Is <TimelineRecord>(t => t.Issues.Where(i => i.Type == IssueType.Warning).Count() == 10)), Times.AtLeastOnce);
}
}
public void FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, DateTime jobStartTimeUtc)
{
Trace.Entering();
ArgUtil.NotNull(jobContext, nameof(jobContext));
// create a new timeline record node for 'Finalize job'
IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), "Complete job", $"{nameof(JobExtension)}_Final", null, null);
using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
{
try
{
context.Start();
context.Debug("Starting: Complete job");
// Evaluate job outputs
if (message.JobOutputs != null && message.JobOutputs.Type != TokenType.Null)
{
try
{
context.Output($"Evaluate and set job outputs");
// Populate env context for each step
Trace.Info("Initialize Env context for evaluating job outputs");
#if OS_WINDOWS
var envContext = new DictionaryContextData();
#else
var envContext = new CaseSensitiveDictionaryContextData();
#endif
context.ExpressionValues["env"] = envContext;
foreach (var pair in context.EnvironmentVariables)
{
envContext[pair.Key] = new StringContextData(pair.Value ?? string.Empty);
}
Trace.Info("Initialize steps context for evaluating job outputs");
context.ExpressionValues["steps"] = context.StepsContext.GetScope(context.ScopeName);
var templateEvaluator = context.ToPipelineTemplateEvaluator();
var outputs = templateEvaluator.EvaluateJobOutput(message.JobOutputs, context.ExpressionValues, context.ExpressionFunctions);
foreach (var output in outputs)
{
if (string.IsNullOrEmpty(output.Value))
{
context.Debug($"Skip output '{output.Key}' since it's empty");
continue;
}
if (!string.Equals(output.Value, HostContext.SecretMasker.MaskSecrets(output.Value)))
{
context.Warning($"Skip output '{output.Key}' since it may contain secret.");
continue;
}
context.Output($"Set output '{output.Key}'");
jobContext.JobOutputs[output.Key] = output.Value;
}
}
catch (Exception ex)
{
context.Result = TaskResult.Failed;
context.Error($"Fail to evaluate job outputs");
context.Error(ex);
jobContext.Result = TaskResultUtil.MergeTaskResults(jobContext.Result, TaskResult.Failed);
}
}
if (context.Variables.GetBoolean(Constants.Variables.Actions.RunnerDebug) ?? false)
{
Trace.Info("Support log upload starting.");
context.Output("Uploading runner diagnostic logs");
IDiagnosticLogManager diagnosticLogManager = HostContext.GetService <IDiagnosticLogManager>();
try
{
diagnosticLogManager.UploadDiagnosticLogs(executionContext: context, parentContext: jobContext, message: message, jobStartTimeUtc: jobStartTimeUtc);
Trace.Info("Support log upload complete.");
context.Output("Completed runner diagnostic log upload");
}
catch (Exception ex)
{
// Log the error but make sure we continue gracefully.
Trace.Info("Error uploading support logs.");
context.Output("Error uploading runner diagnostic logs");
Trace.Error(ex);
}
}
if (_processCleanup)
{
context.Output("Cleaning up orphan processes");
// Only check environment variable for any process that doesn't run before we invoke our process.
Dictionary <int, Process> currentProcesses = SnapshotProcesses();
foreach (var proc in currentProcesses)
{
if (proc.Key == Process.GetCurrentProcess().Id)
{
// skip for current process.
continue;
}
if (_existingProcesses.Contains($"{proc.Key}_{proc.Value.ProcessName}"))
{
Trace.Verbose($"Skip existing process. PID: {proc.Key} ({proc.Value.ProcessName})");
}
else
{
Trace.Info($"Inspecting process environment variables. PID: {proc.Key} ({proc.Value.ProcessName})");
string lookupId = null;
try
{
lookupId = proc.Value.GetEnvironmentVariable(HostContext, Constants.ProcessTrackingId);
}
catch (Exception ex)
{
Trace.Warning($"Ignore exception during read process environment variables: {ex.Message}");
Trace.Verbose(ex.ToString());
}
if (string.Equals(lookupId, _processLookupId, StringComparison.OrdinalIgnoreCase))
{
context.Output($"Terminate orphan process: pid ({proc.Key}) ({proc.Value.ProcessName})");
try
{
proc.Value.Kill();
}
catch (Exception ex)
{
Trace.Error("Catch exception during orphan process cleanup.");
Trace.Error(ex);
}
}
}
}
}
}
catch (Exception ex)
{
// Log and ignore the error from JobExtension finalization.
Trace.Error($"Caught exception from JobExtension finalization: {ex}");
context.Output(ex.Message);
}
finally
{
context.Debug("Finishing: Complete job");
context.Complete();
}
}
}
// Download all required actions.
// Make sure all condition inputs are valid.
// Build up three list of steps for jobrunner (pre-job, job, post-job).
public async Task <List <IStep> > InitializeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message)
{
Trace.Entering();
ArgUtil.NotNull(jobContext, nameof(jobContext));
ArgUtil.NotNull(message, nameof(message));
// Create a new timeline record for 'Set up job'
IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), "Set up job", $"{nameof(JobExtension)}_Init", null, null);
List <IStep> preJobSteps = new List <IStep>();
List <IStep> jobSteps = new List <IStep>();
using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
{
try
{
context.Start();
context.Debug($"Starting: Set up job");
context.Output($"Current runner version: '{BuildConstants.RunnerPackage.Version}'");
var setupInfoFile = HostContext.GetConfigFile(WellKnownConfigFile.SetupInfo);
if (File.Exists(setupInfoFile))
{
Trace.Info($"Load machine setup info from {setupInfoFile}");
try
{
var setupInfo = IOUtil.LoadObject <List <SetupInfo> >(setupInfoFile);
if (setupInfo?.Count > 0)
{
foreach (var info in setupInfo)
{
if (!string.IsNullOrEmpty(info?.Detail))
{
var groupName = info.Group;
if (string.IsNullOrEmpty(groupName))
{
groupName = "Machine Setup Info";
}
context.Output($"##[group]{groupName}");
var multiLines = info.Detail.Replace("\r\n", "\n").TrimEnd('\n').Split('\n');
foreach (var line in multiLines)
{
context.Output(line);
}
context.Output("##[endgroup]");
}
}
}
}
catch (Exception ex)
{
context.Output($"Fail to load and print machine setup info: {ex.Message}");
Trace.Error(ex);
}
}
var repoFullName = context.GetGitHubContext("repository");
ArgUtil.NotNull(repoFullName, nameof(repoFullName));
context.Debug($"Primary repository: {repoFullName}");
// Print proxy setting information for better diagnostic experience
if (!string.IsNullOrEmpty(HostContext.WebProxy.HttpProxyAddress))
{
context.Output($"Runner is running behind proxy server '{HostContext.WebProxy.HttpProxyAddress}' for all HTTP requests.");
}
if (!string.IsNullOrEmpty(HostContext.WebProxy.HttpsProxyAddress))
{
context.Output($"Runner is running behind proxy server '{HostContext.WebProxy.HttpsProxyAddress}' for all HTTPS requests.");
}
// Prepare the workflow directory
context.Output("Prepare workflow directory");
var directoryManager = HostContext.GetService <IPipelineDirectoryManager>();
TrackingConfig trackingConfig = directoryManager.PrepareDirectory(
context,
message.Workspace);
// Set the directory variables
context.Debug("Update context data");
string _workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
context.SetRunnerContext("workspace", Path.Combine(_workDirectory, trackingConfig.PipelineDirectory));
context.SetGitHubContext("workspace", Path.Combine(_workDirectory, trackingConfig.WorkspaceDirectory));
// Temporary hack for GHES alpha
var configurationStore = HostContext.GetService <IConfigurationStore>();
var runnerSettings = configurationStore.GetSettings();
if (string.IsNullOrEmpty(context.GetGitHubContext("server_url")) && !runnerSettings.IsHostedServer && !string.IsNullOrEmpty(runnerSettings.GitHubUrl))
{
var url = new Uri(runnerSettings.GitHubUrl);
var portInfo = url.IsDefaultPort ? string.Empty : $":{url.Port.ToString(CultureInfo.InvariantCulture)}";
context.SetGitHubContext("server_url", $"{url.Scheme}://{url.Host}{portInfo}");
context.SetGitHubContext("api_url", $"{url.Scheme}://{url.Host}{portInfo}/api/v3");
context.SetGitHubContext("graphql_url", $"{url.Scheme}://{url.Host}{portInfo}/api/graphql");
}
// Evaluate the job-level environment variables
context.Debug("Evaluating job-level environment variables");
var templateEvaluator = context.ToPipelineTemplateEvaluator();
foreach (var token in message.EnvironmentVariables)
{
var environmentVariables = templateEvaluator.EvaluateStepEnvironment(token, jobContext.ExpressionValues, jobContext.ExpressionFunctions, VarUtil.EnvironmentVariableKeyComparer);
foreach (var pair in environmentVariables)
{
context.EnvironmentVariables[pair.Key] = pair.Value ?? string.Empty;
context.SetEnvContext(pair.Key, pair.Value ?? string.Empty);
}
}
// Evaluate the job container
context.Debug("Evaluating job container");
var container = templateEvaluator.EvaluateJobContainer(message.JobContainer, jobContext.ExpressionValues, jobContext.ExpressionFunctions);
if (container != null)
{
jobContext.Container = new Container.ContainerInfo(HostContext, container);
}
// Evaluate the job service containers
context.Debug("Evaluating job service containers");
var serviceContainers = templateEvaluator.EvaluateJobServiceContainers(message.JobServiceContainers, jobContext.ExpressionValues, jobContext.ExpressionFunctions);
if (serviceContainers?.Count > 0)
{
foreach (var pair in serviceContainers)
{
var networkAlias = pair.Key;
var serviceContainer = pair.Value;
jobContext.ServiceContainers.Add(new Container.ContainerInfo(HostContext, serviceContainer, false, networkAlias));
}
}
// Evaluate the job defaults
context.Debug("Evaluating job defaults");
foreach (var token in message.Defaults)
{
var defaults = token.AssertMapping("defaults");
if (defaults.Any(x => string.Equals(x.Key.AssertString("defaults key").Value, "run", StringComparison.OrdinalIgnoreCase)))
{
context.JobDefaults["run"] = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
var defaultsRun = defaults.First(x => string.Equals(x.Key.AssertString("defaults key").Value, "run", StringComparison.OrdinalIgnoreCase));
var jobDefaults = templateEvaluator.EvaluateJobDefaultsRun(defaultsRun.Value, jobContext.ExpressionValues, jobContext.ExpressionFunctions);
foreach (var pair in jobDefaults)
{
if (!string.IsNullOrEmpty(pair.Value))
{
context.JobDefaults["run"][pair.Key] = pair.Value;
}
}
}
}
// Build up 2 lists of steps, pre-job, job
// Download actions not already in the cache
Trace.Info("Downloading actions");
var actionManager = HostContext.GetService <IActionManager>();
var prepareResult = await actionManager.PrepareActionsAsync(context, message.Steps);
preJobSteps.AddRange(prepareResult.ContainerSetupSteps);
// Add start-container steps, record and stop-container steps
if (jobContext.Container != null || jobContext.ServiceContainers.Count > 0)
{
var containerProvider = HostContext.GetService <IContainerOperationProvider>();
var containers = new List <Container.ContainerInfo>();
if (jobContext.Container != null)
{
containers.Add(jobContext.Container);
}
containers.AddRange(jobContext.ServiceContainers);
preJobSteps.Add(new JobExtensionRunner(runAsync: containerProvider.StartContainersAsync,
condition: $"{PipelineTemplateConstants.Success}()",
displayName: "Initialize containers",
data: (object)containers));
}
// Add action steps
foreach (var step in message.Steps)
{
if (step.Type == Pipelines.StepType.Action)
{
var action = step as Pipelines.ActionStep;
Trace.Info($"Adding {action.DisplayName}.");
var actionRunner = HostContext.CreateService <IActionRunner>();
actionRunner.Action = action;
actionRunner.Stage = ActionRunStage.Main;
actionRunner.Condition = step.Condition;
var contextData = new Pipelines.ContextData.DictionaryContextData();
if (message.ContextData?.Count > 0)
{
foreach (var pair in message.ContextData)
{
contextData[pair.Key] = pair.Value;
}
}
actionRunner.TryEvaluateDisplayName(contextData, context);
jobSteps.Add(actionRunner);
if (prepareResult.PreStepTracker.TryGetValue(step.Id, out var preStep))
{
Trace.Info($"Adding pre-{action.DisplayName}.");
preStep.TryEvaluateDisplayName(contextData, context);
preStep.DisplayName = $"Pre {preStep.DisplayName}";
preJobSteps.Add(preStep);
}
}
}
var intraActionStates = new Dictionary <Guid, Dictionary <string, string> >();
foreach (var preStep in prepareResult.PreStepTracker)
{
intraActionStates[preStep.Key] = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
}
// Create execution context for pre-job steps
foreach (var step in preJobSteps)
{
if (step is JobExtensionRunner)
{
JobExtensionRunner extensionStep = step as JobExtensionRunner;
ArgUtil.NotNull(extensionStep, extensionStep.DisplayName);
Guid stepId = Guid.NewGuid();
extensionStep.ExecutionContext = jobContext.CreateChild(stepId, extensionStep.DisplayName, null, null, stepId.ToString("N"));
}
else if (step is IActionRunner actionStep)
{
ArgUtil.NotNull(actionStep, step.DisplayName);
Guid stepId = Guid.NewGuid();
actionStep.ExecutionContext = jobContext.CreateChild(stepId, actionStep.DisplayName, stepId.ToString("N"), null, null, intraActionStates[actionStep.Action.Id]);
}
}
// Create execution context for job steps
foreach (var step in jobSteps)
{
if (step is IActionRunner actionStep)
{
ArgUtil.NotNull(actionStep, step.DisplayName);
intraActionStates.TryGetValue(actionStep.Action.Id, out var intraActionState);
actionStep.ExecutionContext = jobContext.CreateChild(actionStep.Action.Id, actionStep.DisplayName, actionStep.Action.Name, actionStep.Action.ScopeName, actionStep.Action.ContextName, intraActionState);
}
}
List <IStep> steps = new List <IStep>();
steps.AddRange(preJobSteps);
steps.AddRange(jobSteps);
// Prepare for orphan process cleanup
_processCleanup = jobContext.Variables.GetBoolean("process.clean") ?? true;
if (_processCleanup)
{
// Set the RUNNER_TRACKING_ID env variable.
Environment.SetEnvironmentVariable(Constants.ProcessTrackingId, _processLookupId);
context.Debug("Collect running processes for tracking orphan processes.");
// Take a snapshot of current running processes
Dictionary <int, Process> processes = SnapshotProcesses();
foreach (var proc in processes)
{
// Pid_ProcessName
_existingProcesses.Add($"{proc.Key}_{proc.Value.ProcessName}");
}
}
return(steps);
}
catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
{
// Log the exception and cancel the JobExtension Initialization.
Trace.Error($"Caught cancellation exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Canceled;
throw;
}
catch (Exception ex)
{
// Log the error and fail the JobExtension Initialization.
Trace.Error($"Caught exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Failed;
throw;
}
finally
{
context.Debug("Finishing: Set up job");
context.Complete();
}
}
}
private bool IsMessageIdentical(Pipelines.AgentJobRequestMessage source, Pipelines.AgentJobRequestMessage target)
{
if (source == null && target == null)
{
return(true);
}
if (source != null && target == null)
{
return(false);
}
if (source == null && target != null)
{
return(false);
}
if (JsonUtility.ToString(source.JobContainer) != JsonUtility.ToString(target.JobContainer))
{
return(false);
}
if (source.JobDisplayName != target.JobDisplayName)
{
return(false);
}
if (source.JobId != target.JobId)
{
return(false);
}
if (source.JobName != target.JobName)
{
return(false);
}
if (source.MaskHints.Count != target.MaskHints.Count)
{
return(false);
}
if (source.MessageType != target.MessageType)
{
return(false);
}
if (source.Plan.PlanId != target.Plan.PlanId)
{
return(false);
}
if (source.RequestId != target.RequestId)
{
return(false);
}
if (source.Resources.Endpoints.Count != target.Resources.Endpoints.Count)
{
return(false);
}
if (source.Steps.Count != target.Steps.Count)
{
return(false);
}
if (source.Variables.Count != target.Variables.Count)
{
return(false);
}
return(true);
}
private async Task RunAsync(Pipelines.AgentJobRequestMessage message, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken, CancellationToken workerCancelTimeoutKillToken)
{
if (previousJobDispatch != null)
{
Trace.Verbose($"Make sure the previous job request {previousJobDispatch.JobId} has successfully finished on worker.");
await EnsureDispatchFinished(previousJobDispatch);
}
else
{
Trace.Verbose($"This is the first job request.");
}
var term = HostContext.GetService <ITerminal>();
term.WriteLine($"{DateTime.UtcNow:u}: Running job: {message.JobDisplayName}");
// first job request renew succeed.
TaskCompletionSource <int> firstJobRequestRenewed = new TaskCompletionSource <int>();
var notification = HostContext.GetService <IJobNotification>();
// lock renew cancellation token.
using (var lockRenewalTokenSource = new CancellationTokenSource())
using (var workerProcessCancelTokenSource = new CancellationTokenSource())
{
long requestId = message.RequestId;
Guid lockToken = Guid.Empty; // lockToken has never been used, keep this here of compat
// start renew job request
Trace.Info($"Start renew job request {requestId} for job {message.JobId}.");
Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, firstJobRequestRenewed, lockRenewalTokenSource.Token);
// wait till first renew succeed or job request is canceled
// not even start worker if the first renew fail
await Task.WhenAny(firstJobRequestRenewed.Task, renewJobRequest, Task.Delay(-1, jobRequestCancellationToken));
if (renewJobRequest.IsCompleted)
{
// renew job request task complete means we run out of retry for the first job request renew.
Trace.Info($"Unable to renew job request for job {message.JobId} for the first time, stop dispatching job to worker.");
return;
}
if (jobRequestCancellationToken.IsCancellationRequested)
{
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request with result Cancelled
await CompleteJobRequestAsync(_poolId, message, lockToken, TaskResult.Canceled);
return;
}
HostContext.WritePerfCounter($"JobRequestRenewed_{requestId.ToString()}");
Task <int> workerProcessTask = null;
object _outputLock = new object();
List <string> workerOutput = new List <string>();
using (var processChannel = HostContext.CreateService <IProcessChannel>())
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
// Start the process channel.
// It's OK if StartServer bubbles an execption after the worker process has already started.
// The worker will shutdown after 30 seconds if it hasn't received the job message.
processChannel.StartServer(
// Delegate to start the child process.
startProcess: (string pipeHandleOut, string pipeHandleIn) =>
{
// Validate args.
ArgUtil.NotNullOrEmpty(pipeHandleOut, nameof(pipeHandleOut));
ArgUtil.NotNullOrEmpty(pipeHandleIn, nameof(pipeHandleIn));
// Save STDOUT from worker, worker will use STDOUT report unhandle exception.
processInvoker.OutputDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stdout)
{
if (!string.IsNullOrEmpty(stdout.Data))
{
lock (_outputLock)
{
workerOutput.Add(stdout.Data);
}
}
};
// Save STDERR from worker, worker will use STDERR on crash.
processInvoker.ErrorDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stderr)
{
if (!string.IsNullOrEmpty(stderr.Data))
{
lock (_outputLock)
{
workerOutput.Add(stderr.Data);
}
}
};
// Start the child process.
HostContext.WritePerfCounter("StartingWorkerProcess");
var assemblyDirectory = HostContext.GetDirectory(WellKnownDirectory.Bin);
string workerFileName = Path.Combine(assemblyDirectory, _workerProcessName);
workerProcessTask = processInvoker.ExecuteAsync(
workingDirectory: assemblyDirectory,
fileName: workerFileName,
arguments: "spawnclient " + pipeHandleOut + " " + pipeHandleIn,
environment: null,
requireExitCodeZero: false,
outputEncoding: null,
killProcessOnCancel: true,
redirectStandardIn: null,
inheritConsoleHandler: false,
keepStandardInOpen: false,
highPriorityProcess: true,
cancellationToken: workerProcessCancelTokenSource.Token);
});
// Send the job request message.
// Kill the worker process if sending the job message times out. The worker
// process may have successfully received the job message.
try
{
Trace.Info($"Send job request message to worker for job {message.JobId}.");
HostContext.WritePerfCounter($"RunnerSendingJobToWorker_{message.JobId}");
using (var csSendJobRequest = new CancellationTokenSource(_channelTimeout))
{
await processChannel.SendAsync(
messageType : MessageType.NewJobRequest,
body : JsonUtility.ToString(message),
cancellationToken : csSendJobRequest.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
// timeout 30 sec. kill worker.
Trace.Info($"Job request message sending for job {message.JobId} been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// not finish the job request since the job haven't run on worker at all, we will not going to set a result to server.
return;
}
// we get first jobrequest renew succeed and start the worker process with the job message.
// send notification to machine provisioner.
var systemConnection = message.Resources.Endpoints.SingleOrDefault(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
var accessToken = systemConnection?.Authorization?.Parameters["AccessToken"];
notification.JobStarted(message.JobId, accessToken, systemConnection.Url);
HostContext.WritePerfCounter($"SentJobToWorker_{requestId.ToString()}");
try
{
TaskResult resultOnAbandonOrCancel = TaskResult.Succeeded;
// wait for renewlock, worker process or cancellation token been fired.
var completedTask = await Task.WhenAny(renewJobRequest, workerProcessTask, Task.Delay(-1, jobRequestCancellationToken));
if (completedTask == workerProcessTask)
{
// worker finished successfully, complete job request with result, attach unhandled exception reported by worker, stop renew lock, job has finished.
int returnCode = await workerProcessTask;
Trace.Info($"Worker finished for job {message.JobId}. Code: " + returnCode);
string detailInfo = null;
if (!TaskResultUtil.IsValidReturnCode(returnCode))
{
detailInfo = string.Join(Environment.NewLine, workerOutput);
Trace.Info($"Return code {returnCode} indicate worker encounter an unhandled exception or app crash, attach worker stdout/stderr to JobRequest result.");
await LogWorkerProcessUnhandledException(message, detailInfo);
}
TaskResult result = TaskResultUtil.TranslateFromReturnCode(returnCode);
Trace.Info($"finish job request for job {message.JobId} with result: {result}");
term.WriteLine($"{DateTime.UtcNow:u}: Job {message.JobDisplayName} completed with result: {result}");
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request
await CompleteJobRequestAsync(_poolId, message, lockToken, result, detailInfo);
// print out unhandled exception happened in worker after we complete job request.
// when we run out of disk space, report back to server has higher priority.
if (!string.IsNullOrEmpty(detailInfo))
{
Trace.Error("Unhandled exception happened in worker:");
Trace.Error(detailInfo);
}
return;
}
else if (completedTask == renewJobRequest)
{
resultOnAbandonOrCancel = TaskResult.Abandoned;
}
else
{
resultOnAbandonOrCancel = TaskResult.Canceled;
}
// renew job request completed or job request cancellation token been fired for RunAsync(jobrequestmessage)
// cancel worker gracefully first, then kill it after worker cancel timeout
try
{
Trace.Info($"Send job cancellation message to worker for job {message.JobId}.");
using (var csSendCancel = new CancellationTokenSource(_channelTimeout))
{
var messageType = MessageType.CancelRequest;
if (HostContext.RunnerShutdownToken.IsCancellationRequested)
{
switch (HostContext.RunnerShutdownReason)
{
case ShutdownReason.UserCancelled:
messageType = MessageType.RunnerShutdown;
break;
case ShutdownReason.OperatingSystemShutdown:
messageType = MessageType.OperatingSystemShutdown;
break;
}
}
await processChannel.SendAsync(
messageType : messageType,
body : string.Empty,
cancellationToken : csSendCancel.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
Trace.Info($"Job cancel message sending for job {message.JobId} been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
}
// wait worker to exit
// if worker doesn't exit within timeout, then kill worker.
completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, workerCancelTimeoutKillToken));
// worker haven't exit within cancellation timeout.
if (completedTask != workerProcessTask)
{
Trace.Info($"worker process for job {message.JobId} haven't exit within cancellation timout, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
}
Trace.Info($"finish job request for job {message.JobId} with result: {resultOnAbandonOrCancel}");
term.WriteLine($"{DateTime.UtcNow:u}: Job {message.JobDisplayName} completed with result: {resultOnAbandonOrCancel}");
// complete job request with cancel result, stop renew lock, job has finished.
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request
await CompleteJobRequestAsync(_poolId, message, lockToken, resultOnAbandonOrCancel);
}
finally
{
// This should be the last thing to run so we don't notify external parties until actually finished
await notification.JobCompleted(message.JobId);
}
}
}
}
private async Task <TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, TaskResult?taskResult = null)
{
jobContext.Debug($"Finishing: {message.JobDisplayName}");
TaskResult result = jobContext.Complete(taskResult);
try
{
await ShutdownQueue(throwOnFailure : true);
}
catch (Exception ex)
{
Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(_jobServerQueue.ShutdownAsync)}");
Trace.Error("This indicate a failure during publish output variables. Fail the job to prevent unexpected job outputs.");
Trace.Error(ex);
result = TaskResultUtil.MergeTaskResults(result, TaskResult.Failed);
}
// Clean TEMP after finish process jobserverqueue, since there might be a pending fileupload still use the TEMP dir.
_tempDirectoryManager?.CleanupTempDirectory();
if (!jobContext.Features.HasFlag(PlanFeatures.JobCompletedPlanEvent))
{
Trace.Info($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}");
return(result);
}
Trace.Info("Raising job completed event.");
var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result);
var completeJobRetryLimit = 5;
var exceptions = new List <Exception>();
while (completeJobRetryLimit-- > 0)
{
try
{
await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, default(CancellationToken));
return(result);
}
catch (TaskOrchestrationPlanNotFoundException ex)
{
Trace.Error($"TaskOrchestrationPlanNotFoundException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
Trace.Error(ex);
return(TaskResult.Failed);
}
catch (TaskOrchestrationPlanSecurityException ex)
{
Trace.Error($"TaskOrchestrationPlanSecurityException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
Trace.Error(ex);
return(TaskResult.Failed);
}
catch (Exception ex)
{
Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}.");
Trace.Error(ex);
exceptions.Add(ex);
}
// delay 5 seconds before next retry.
await Task.Delay(TimeSpan.FromSeconds(5));
}
// rethrow exceptions from all attempts.
throw new AggregateException(exceptions);
}
public async Task <TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Resources, nameof(message.Resources));
ArgUtil.NotNull(message.Variables, nameof(message.Variables));
ArgUtil.NotNull(message.Steps, nameof(message.Steps));
Trace.Info("Job ID {0}", message.JobId);
DateTime jobStartTimeUtc = DateTime.UtcNow;
ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
// Setup the job server and job server queue.
var jobServer = HostContext.GetService <IJobServer>();
VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection);
Uri jobServerUrl = systemConnection.Url;
Trace.Info($"Creating job server with URL: {jobServerUrl}");
// jobServerQueue is the throttling reporter.
_jobServerQueue = HostContext.GetService <IJobServerQueue>();
VssConnection jobConnection = VssUtil.CreateConnection(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(_jobServerQueue) });
await jobServer.ConnectAsync(jobConnection);
_jobServerQueue.Start(message);
HostContext.WritePerfCounter($"WorkerJobServerQueueStarted_{message.RequestId.ToString()}");
IExecutionContext jobContext = null;
CancellationTokenRegistration?runnerShutdownRegistration = null;
try
{
// Create the job execution context.
jobContext = HostContext.CreateService <IExecutionContext>();
jobContext.InitializeJob(message, jobRequestCancellationToken);
Trace.Info("Starting the job execution context.");
jobContext.Start();
jobContext.Debug($"Starting: {message.JobDisplayName}");
runnerShutdownRegistration = HostContext.RunnerShutdownToken.Register(() =>
{
// log an issue, then runner get shutdown by Ctrl-C or Ctrl-Break.
// the server will use Ctrl-Break to tells the runner that operating system is shutting down.
string errorMessage;
switch (HostContext.RunnerShutdownReason)
{
case ShutdownReason.UserCancelled:
errorMessage = "The runner has received a shutdown signal. This can happen when the runner service is stopped, or a manually started runner is canceled.";
break;
case ShutdownReason.OperatingSystemShutdown:
errorMessage = $"Operating system is shutting down for computer '{Environment.MachineName}'";
break;
default:
throw new ArgumentException(HostContext.RunnerShutdownReason.ToString(), nameof(HostContext.RunnerShutdownReason));
}
jobContext.AddIssue(new Issue()
{
Type = IssueType.Error, Message = errorMessage
});
});
// Validate directory permissions.
string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
Trace.Info($"Validating directory permissions for: '{workDirectory}'");
try
{
Directory.CreateDirectory(workDirectory);
IOUtil.ValidateExecutePermission(workDirectory);
}
catch (Exception ex)
{
Trace.Error(ex);
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
if (jobContext.WriteDebug)
{
jobContext.SetRunnerContext("debug", "1");
}
jobContext.SetRunnerContext("os", VarUtil.OS);
string toolsDirectory = HostContext.GetDirectory(WellKnownDirectory.Tools);
Directory.CreateDirectory(toolsDirectory);
jobContext.SetRunnerContext("tool_cache", toolsDirectory);
// Setup TEMP directories
_tempDirectoryManager = HostContext.GetService <ITempDirectoryManager>();
_tempDirectoryManager.InitializeTempDirectory(jobContext);
// // Expand container properties
// jobContext.Container?.ExpandProperties(jobContext.Variables);
// foreach (var sidecar in jobContext.SidecarContainers)
// {
// sidecar.ExpandProperties(jobContext.Variables);
// }
// Get the job extension.
Trace.Info("Getting job extension.");
IJobExtension jobExtension = HostContext.CreateService <IJobExtension>();
List <IStep> jobSteps = null;
try
{
Trace.Info("Initialize job. Getting all job steps.");
jobSteps = await jobExtension.InitializeJob(jobContext, message);
}
catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
{
// set the job to canceled
// don't log error issue to job ExecutionContext, since server owns the job level issue
Trace.Error($"Job is canceled during initialize.");
Trace.Error($"Caught exception: {ex}");
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled));
}
catch (Exception ex)
{
// set the job to failed.
// don't log error issue to job ExecutionContext, since server owns the job level issue
Trace.Error($"Job initialize failed.");
Trace.Error($"Caught exception from {nameof(jobExtension.InitializeJob)}: {ex}");
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
// trace out all steps
Trace.Info($"Total job steps: {jobSteps.Count}.");
Trace.Verbose($"Job steps: '{string.Join(", ", jobSteps.Select(x => x.DisplayName))}'");
HostContext.WritePerfCounter($"WorkerJobInitialized_{message.RequestId.ToString()}");
// Run all job steps
Trace.Info("Run all job steps.");
var stepsRunner = HostContext.GetService <IStepsRunner>();
try
{
foreach (var step in jobSteps)
{
jobContext.JobSteps.Enqueue(step);
}
await stepsRunner.RunAsync(jobContext);
}
catch (Exception ex)
{
// StepRunner should never throw exception out.
// End up here mean there is a bug in StepRunner
// Log the error and fail the job.
Trace.Error($"Caught exception from job steps {nameof(StepsRunner)}: {ex}");
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
finally
{
Trace.Info("Finalize job.");
jobExtension.FinalizeJob(jobContext, message, jobStartTimeUtc);
}
Trace.Info($"Job result after all job steps finish: {jobContext.Result ?? TaskResult.Succeeded}");
Trace.Info("Completing the job execution context.");
return(await CompleteJobAsync(jobServer, jobContext, message));
}
finally
{
if (runnerShutdownRegistration != null)
{
runnerShutdownRegistration.Value.Dispose();
runnerShutdownRegistration = null;
}
await ShutdownQueue(throwOnFailure : false);
}
}
public void RegisterPostJobAction_NotRegisterPostTwice()
{
using (TestHostContext hc = CreateTestContext())
{
// Arrange: Create a job request message.
TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
TimelineReference timeline = new TimelineReference();
Guid jobId = Guid.NewGuid();
string jobName = "some job name";
var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary <string, VariableValue>(), new List <MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List <Pipelines.ActionStep>(), null, null, null, null);
jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
{
Alias = Pipelines.PipelineConstants.SelfAlias,
Id = "github",
Version = "sha1"
});
jobRequest.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData();
jobRequest.Variables["ACTIONS_STEP_DEBUG"] = "true";
// Arrange: Setup the paging logger.
var pagingLogger1 = new Mock <IPagingLogger>();
var pagingLogger2 = new Mock <IPagingLogger>();
var pagingLogger3 = new Mock <IPagingLogger>();
var pagingLogger4 = new Mock <IPagingLogger>();
var pagingLogger5 = new Mock <IPagingLogger>();
var jobServerQueue = new Mock <IJobServerQueue>();
jobServerQueue.Setup(x => x.QueueTimelineRecordUpdate(It.IsAny <Guid>(), It.IsAny <TimelineRecord>()));
jobServerQueue.Setup(x => x.QueueWebConsoleLine(It.IsAny <Guid>(), It.IsAny <string>(), It.IsAny <long?>())).Callback((Guid id, string msg, long?lineNumber) => { hc.GetTrace().Info(msg); });
var actionRunner1 = new ActionRunner();
actionRunner1.Initialize(hc);
var actionRunner2 = new ActionRunner();
actionRunner2.Initialize(hc);
hc.EnqueueInstance(pagingLogger1.Object);
hc.EnqueueInstance(pagingLogger2.Object);
hc.EnqueueInstance(pagingLogger3.Object);
hc.EnqueueInstance(pagingLogger4.Object);
hc.EnqueueInstance(pagingLogger5.Object);
hc.EnqueueInstance(actionRunner1 as IActionRunner);
hc.EnqueueInstance(actionRunner2 as IActionRunner);
hc.SetSingleton(jobServerQueue.Object);
var jobContext = new Runner.Worker.ExecutionContext();
jobContext.Initialize(hc);
// Act.
jobContext.InitializeJob(jobRequest, CancellationToken.None);
var action1 = jobContext.CreateChild(Guid.NewGuid(), "action_1_pre", "action_1_pre", null, null, 0);
var action2 = jobContext.CreateChild(Guid.NewGuid(), "action_1_main", "action_1_main", null, null, 0);
var actionId = Guid.NewGuid();
var postRunner1 = hc.CreateService <IActionRunner>();
postRunner1.Action = new Pipelines.ActionStep()
{
Id = actionId, Name = "post1", DisplayName = "Test 1", Reference = new Pipelines.RepositoryPathReference()
{
Name = "actions/action"
}
};
postRunner1.Stage = ActionRunStage.Post;
postRunner1.Condition = "always()";
postRunner1.DisplayName = "post1";
var postRunner2 = hc.CreateService <IActionRunner>();
postRunner2.Action = new Pipelines.ActionStep()
{
Id = actionId, Name = "post2", DisplayName = "Test 2", Reference = new Pipelines.RepositoryPathReference()
{
Name = "actions/action"
}
};
postRunner2.Stage = ActionRunStage.Post;
postRunner2.Condition = "always()";
postRunner2.DisplayName = "post2";
action1.RegisterPostJobStep(postRunner1);
action2.RegisterPostJobStep(postRunner2);
Assert.NotNull(jobContext.JobSteps);
Assert.NotNull(jobContext.PostJobSteps);
Assert.Equal(1, jobContext.PostJobSteps.Count);
var post1 = jobContext.PostJobSteps.Pop();
Assert.Equal("post1", (post1 as IActionRunner).Action.Name);
Assert.Equal(ActionRunStage.Post, (post1 as IActionRunner).Stage);
Assert.Equal("always()", (post1 as IActionRunner).Condition);
}
}
public void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token)
{
// Validation
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Resources, nameof(message.Resources));
ArgUtil.NotNull(message.Variables, nameof(message.Variables));
ArgUtil.NotNull(message.Plan, nameof(message.Plan));
_cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token);
// Features
Features = PlanUtil.GetFeatures(message.Plan);
// Endpoints
Endpoints = message.Resources.Endpoints;
// Variables
Variables = new Variables(HostContext, message.Variables);
// Environment variables shared across all actions
EnvironmentVariables = new Dictionary <string, string>(VarUtil.EnvironmentVariableKeyComparer);
// Job defaults shared across all actions
JobDefaults = new Dictionary <string, IDictionary <string, string> >(StringComparer.OrdinalIgnoreCase);
// Job Outputs
JobOutputs = new Dictionary <string, VariableValue>(StringComparer.OrdinalIgnoreCase);
// Service container info
ServiceContainers = new List <ContainerInfo>();
// Steps context (StepsRunner manages adding the scoped steps context)
StepsContext = new StepsContext();
// Scopes
Scopes = new Dictionary <String, ContextScope>(StringComparer.OrdinalIgnoreCase);
if (message.Scopes?.Count > 0)
{
foreach (var scope in message.Scopes)
{
Scopes[scope.Name] = scope;
}
}
// File table
FileTable = new List <String>(message.FileTable ?? new string[0]);
// Expression values
if (message.ContextData?.Count > 0)
{
foreach (var pair in message.ContextData)
{
ExpressionValues[pair.Key] = pair.Value;
}
}
ExpressionValues["secrets"] = Variables.ToSecretsContext();
ExpressionValues["runner"] = new RunnerContext();
ExpressionValues["job"] = new JobContext();
Trace.Info("Initialize GitHub context");
var githubAccessToken = new StringContextData(Variables.Get("system.github.token"));
var base64EncodedToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{githubAccessToken}"));
HostContext.SecretMasker.AddValue(base64EncodedToken);
var githubJob = Variables.Get("system.github.job");
var githubContext = new GitHubContext();
githubContext["token"] = githubAccessToken;
if (!string.IsNullOrEmpty(githubJob))
{
githubContext["job"] = new StringContextData(githubJob);
}
var githubDictionary = ExpressionValues["github"].AssertDictionary("github");
foreach (var pair in githubDictionary)
{
githubContext[pair.Key] = pair.Value;
}
ExpressionValues["github"] = githubContext;
Trace.Info("Initialize Env context");
#if OS_WINDOWS
ExpressionValues["env"] = new DictionaryContextData();
#else
ExpressionValues["env"] = new CaseSensitiveDictionaryContextData();
#endif
// Prepend Path
PrependPath = new List <string>();
// JobSteps for job ExecutionContext
JobSteps = new Queue <IStep>();
// PostJobSteps for job ExecutionContext
PostJobSteps = new Stack <IStep>();
// Job timeline record.
InitializeTimelineRecord(
timelineId: message.Timeline.Id,
timelineRecordId: message.JobId,
parentTimelineRecordId: null,
recordType: ExecutionContextType.Job,
displayName: message.JobDisplayName,
refName: message.JobName,
order: null); // The job timeline record's order is set by server.
// Logger (must be initialized before writing warnings).
_logger = HostContext.CreateService <IPagingLogger>();
_logger.Setup(_mainTimelineId, _record.Id);
// Initialize 'echo on action command success' property, default to false, unless Step_Debug is set
EchoOnActionCommand = Variables.Step_Debug ?? false;
// Verbosity (from GitHub.Step_Debug).
WriteDebug = Variables.Step_Debug ?? false;
// Hook up JobServerQueueThrottling event, we will log warning on server tarpit.
_jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived;
}
public void UploadDiagnosticLogs(IExecutionContext executionContext,
IExecutionContext parentContext,
Pipelines.AgentJobRequestMessage message,
DateTime jobStartTimeUtc)
{
executionContext.Debug("Starting diagnostic file upload.");
// Setup folders
// \_layout\_work\_temp\[jobname-support]
executionContext.Debug("Setting up diagnostic log folders.");
string tempDirectory = HostContext.GetDirectory(WellKnownDirectory.Temp);
ArgUtil.Directory(tempDirectory, nameof(tempDirectory));
string supportRootFolder = Path.Combine(tempDirectory, message.JobName + "-support");
Directory.CreateDirectory(supportRootFolder);
// \_layout\_work\_temp\[jobname-support]\files
executionContext.Debug("Creating diagnostic log files folder.");
string supportFilesFolder = Path.Combine(supportRootFolder, "files");
Directory.CreateDirectory(supportFilesFolder);
// Create the environment file
// \_layout\_work\_temp\[jobname-support]\files\environment.txt
var configurationStore = HostContext.GetService <IConfigurationStore>();
RunnerSettings settings = configurationStore.GetSettings();
int runnerId = settings.AgentId;
string runnerName = settings.AgentName;
int poolId = settings.PoolId;
// Copy worker diagnostic log files
List <string> workerDiagnosticLogFiles = GetWorkerDiagnosticLogFiles(HostContext.GetDirectory(WellKnownDirectory.Diag), jobStartTimeUtc);
executionContext.Debug($"Copying {workerDiagnosticLogFiles.Count()} worker diagnostic logs.");
foreach (string workerLogFile in workerDiagnosticLogFiles)
{
ArgUtil.File(workerLogFile, nameof(workerLogFile));
string destination = Path.Combine(supportFilesFolder, Path.GetFileName(workerLogFile));
File.Copy(workerLogFile, destination);
}
// Copy runner diag log files
List <string> runnerDiagnosticLogFiles = GetRunnerDiagnosticLogFiles(HostContext.GetDirectory(WellKnownDirectory.Diag), jobStartTimeUtc);
executionContext.Debug($"Copying {runnerDiagnosticLogFiles.Count()} runner diagnostic logs.");
foreach (string runnerLogFile in runnerDiagnosticLogFiles)
{
ArgUtil.File(runnerLogFile, nameof(runnerLogFile));
string destination = Path.Combine(supportFilesFolder, Path.GetFileName(runnerLogFile));
File.Copy(runnerLogFile, destination);
}
executionContext.Debug("Zipping diagnostic files.");
string buildNumber = executionContext.Variables.Build_Number ?? "UnknownBuildNumber";
string buildName = $"Build {buildNumber}";
string phaseName = executionContext.Variables.System_PhaseDisplayName ?? "UnknownPhaseName";
// zip the files
string diagnosticsZipFileName = $"{buildName}-{phaseName}.zip";
string diagnosticsZipFilePath = Path.Combine(supportRootFolder, diagnosticsZipFileName);
ZipFile.CreateFromDirectory(supportFilesFolder, diagnosticsZipFilePath);
// upload the json metadata file
executionContext.Debug("Uploading diagnostic metadata file.");
string metadataFileName = $"diagnostics-{buildName}-{phaseName}.json";
string metadataFilePath = Path.Combine(supportFilesFolder, metadataFileName);
string phaseResult = GetTaskResultAsString(executionContext.Result);
IOUtil.SaveObject(new DiagnosticLogMetadata(runnerName, runnerId, poolId, phaseName, diagnosticsZipFileName, phaseResult), metadataFilePath);
// TODO: Remove the parentContext Parameter and replace this with executioncontext. Currently a bug exists where these files do not upload correctly using that context.
parentContext.QueueAttachFile(type: CoreAttachmentType.DiagnosticLog, name: metadataFileName, filePath: metadataFilePath);
parentContext.QueueAttachFile(type: CoreAttachmentType.DiagnosticLog, name: diagnosticsZipFileName, filePath: diagnosticsZipFilePath);
executionContext.Debug("Diagnostic file upload complete.");
}
private TestHostContext CreateTestContext([CallerMemberName] String testName = "")
{
var hc = new TestHostContext(this, testName);
_jobEc = new Runner.Worker.ExecutionContext();
_config = new Mock <IConfigurationStore>();
_extensions = new Mock <IExtensionManager>();
_jobExtension = new Mock <IJobExtension>();
_jobServer = new Mock <IJobServer>();
_jobServerQueue = new Mock <IJobServerQueue>();
_stepRunner = new Mock <IStepsRunner>();
_logger = new Mock <IPagingLogger>();
_temp = new Mock <ITempDirectoryManager>();
_diagnosticLogManager = new Mock <IDiagnosticLogManager>();
if (_tokenSource != null)
{
_tokenSource.Dispose();
_tokenSource = null;
}
_tokenSource = new CancellationTokenSource();
var expressionManager = new ExpressionManager();
expressionManager.Initialize(hc);
hc.SetSingleton <IExpressionManager>(expressionManager);
_jobRunner = new JobRunner();
_jobRunner.Initialize(hc);
TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
TimelineReference timeline = new Timeline(Guid.NewGuid());
Guid jobId = Guid.NewGuid();
_message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, testName, testName, null, null, null, new Dictionary <string, VariableValue>(), new List <MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List <Pipelines.ActionStep>(), null);
_message.Variables[Constants.Variables.System.Culture] = "en-US";
_message.Resources.Endpoints.Add(new ServiceEndpoint()
{
Name = WellKnownServiceEndpointNames.SystemVssConnection,
Url = new Uri("https://pipelines.actions.githubusercontent.com"),
Authorization = new EndpointAuthorization()
{
Scheme = "Test",
Parameters =
{
{ "AccessToken", "token" }
}
},
});
_message.Resources.Repositories.Add(new Pipelines.RepositoryResource()
{
Alias = Pipelines.PipelineConstants.SelfAlias,
Id = "github",
Version = "sha1"
});
_message.ContextData.Add("github", new Pipelines.ContextData.DictionaryContextData());
_initResult.Clear();
_jobExtension.Setup(x => x.InitializeJob(It.IsAny <IExecutionContext>(), It.IsAny <Pipelines.AgentJobRequestMessage>())).
Returns(Task.FromResult(_initResult));
var settings = new RunnerSettings
{
AgentId = 1,
AgentName = "agent1",
ServerUrl = "https://pipelines.actions.githubusercontent.com",
WorkFolder = "_work",
};
_config.Setup(x => x.GetSettings())
.Returns(settings);
_logger.Setup(x => x.Setup(It.IsAny <Guid>(), It.IsAny <Guid>()));
hc.SetSingleton(_config.Object);
hc.SetSingleton(_jobServer.Object);
hc.SetSingleton(_jobServerQueue.Object);
hc.SetSingleton(_stepRunner.Object);
hc.SetSingleton(_extensions.Object);
hc.SetSingleton(_temp.Object);
hc.SetSingleton(_diagnosticLogManager.Object);
hc.EnqueueInstance <IExecutionContext>(_jobEc);
hc.EnqueueInstance <IPagingLogger>(_logger.Object);
hc.EnqueueInstance <IJobExtension>(_jobExtension.Object);
return(hc);
}
public async Task <TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Resources, nameof(message.Resources));
ArgUtil.NotNull(message.Variables, nameof(message.Variables));
ArgUtil.NotNull(message.Steps, nameof(message.Steps));
Trace.Info("Job ID {0}", message.JobId);
DateTime jobStartTimeUtc = DateTime.UtcNow;
ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
// Setup the job server and job server queue.
var jobServer = HostContext.GetService <IJobServer>();
VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection);
Uri jobServerUrl = systemConnection.Url;
Trace.Info($"Creating job server with URL: {jobServerUrl}");
// jobServerQueue is the throttling reporter.
_jobServerQueue = HostContext.GetService <IJobServerQueue>();
VssConnection jobConnection = VssUtil.CreateConnection(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(_jobServerQueue) });
await jobServer.ConnectAsync(jobConnection);
_jobServerQueue.Start(message);
HostContext.WritePerfCounter($"WorkerJobServerQueueStarted_{message.RequestId.ToString()}");
IExecutionContext jobContext = null;
CancellationTokenRegistration?runnerShutdownRegistration = null;
try
{
// Create the job execution context.
jobContext = HostContext.CreateService <IExecutionContext>();
jobContext.InitializeJob(message, jobRequestCancellationToken);
Trace.Info("Starting the job execution context.");
jobContext.Start();
jobContext.Debug($"Starting: {message.JobDisplayName}");
// RUST: If the event type is not allowed exit the job before anything is run.
var rustExpectedEvent = System.Environment.GetEnvironmentVariable("RUST_WHITELISTED_EVENT_NAME");
if (rustExpectedEvent != null)
{
var rustGitHubContext = (Pipelines.ContextData.DictionaryContextData)message.ContextData["github"];
var rustEventName = rustGitHubContext["event_name"].ToString();
if (rustEventName != rustExpectedEvent)
{
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled));
}
}
runnerShutdownRegistration = HostContext.RunnerShutdownToken.Register(() =>
{
// log an issue, then runner get shutdown by Ctrl-C or Ctrl-Break.
// the server will use Ctrl-Break to tells the runner that operating system is shutting down.
string errorMessage;
switch (HostContext.RunnerShutdownReason)
{
case ShutdownReason.UserCancelled:
errorMessage = "The runner has received a shutdown signal. This can happen when the runner service is stopped, or a manually started runner is canceled.";
break;
case ShutdownReason.OperatingSystemShutdown:
errorMessage = $"Operating system is shutting down for computer '{Environment.MachineName}'";
break;
default:
throw new ArgumentException(HostContext.RunnerShutdownReason.ToString(), nameof(HostContext.RunnerShutdownReason));
}
jobContext.AddIssue(new Issue()
{
Type = IssueType.Error, Message = errorMessage
});
});
// Validate directory permissions.
string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
Trace.Info($"Validating directory permissions for: '{workDirectory}'");
try
{
Directory.CreateDirectory(workDirectory);
IOUtil.ValidateExecutePermission(workDirectory);
}
catch (Exception ex)
{
Trace.Error(ex);
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
if (jobContext.Global.WriteDebug)
{
jobContext.SetRunnerContext("debug", "1");
}
jobContext.SetRunnerContext("os", VarUtil.OS);
jobContext.SetRunnerContext("arch", VarUtil.OSArchitecture);
var runnerSettings = HostContext.GetService <IConfigurationStore>().GetSettings();
jobContext.SetRunnerContext("name", runnerSettings.AgentName);
string toolsDirectory = HostContext.GetDirectory(WellKnownDirectory.Tools);
Directory.CreateDirectory(toolsDirectory);
jobContext.SetRunnerContext("tool_cache", toolsDirectory);
// Setup TEMP directories
_tempDirectoryManager = HostContext.GetService <ITempDirectoryManager>();
_tempDirectoryManager.InitializeTempDirectory(jobContext);
// Get the job extension.
Trace.Info("Getting job extension.");
IJobExtension jobExtension = HostContext.CreateService <IJobExtension>();
List <IStep> jobSteps = null;
try
{
Trace.Info("Initialize job. Getting all job steps.");
jobSteps = await jobExtension.InitializeJob(jobContext, message);
}
catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
{
// set the job to canceled
// don't log error issue to job ExecutionContext, since server owns the job level issue
Trace.Error($"Job is canceled during initialize.");
Trace.Error($"Caught exception: {ex}");
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled));
}
catch (Exception ex)
{
// set the job to failed.
// don't log error issue to job ExecutionContext, since server owns the job level issue
Trace.Error($"Job initialize failed.");
Trace.Error($"Caught exception from {nameof(jobExtension.InitializeJob)}: {ex}");
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
// trace out all steps
Trace.Info($"Total job steps: {jobSteps.Count}.");
Trace.Verbose($"Job steps: '{string.Join(", ", jobSteps.Select(x => x.DisplayName))}'");
HostContext.WritePerfCounter($"WorkerJobInitialized_{message.RequestId.ToString()}");
if (systemConnection.Data.TryGetValue("GenerateIdTokenUrl", out var generateIdTokenUrl) &&
!string.IsNullOrEmpty(generateIdTokenUrl))
{
// Server won't issue ID_TOKEN for non-inprogress job.
// If the job is trying to use OIDC feature, we want the job to be marked as in-progress before running any customer's steps as much as we can.
// Timeline record update background process runs every 500ms, so delay 1000ms is enough for most of the cases
Trace.Info($"Waiting for job to be marked as started.");
await Task.WhenAny(_jobServerQueue.JobRecordUpdated.Task, Task.Delay(1000));
}
// Run all job steps
Trace.Info("Run all job steps.");
var stepsRunner = HostContext.GetService <IStepsRunner>();
try
{
foreach (var step in jobSteps)
{
jobContext.JobSteps.Enqueue(step);
}
await stepsRunner.RunAsync(jobContext);
}
catch (Exception ex)
{
// StepRunner should never throw exception out.
// End up here mean there is a bug in StepRunner
// Log the error and fail the job.
Trace.Error($"Caught exception from job steps {nameof(StepsRunner)}: {ex}");
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
finally
{
Trace.Info("Finalize job.");
jobExtension.FinalizeJob(jobContext, message, jobStartTimeUtc);
}
Trace.Info($"Job result after all job steps finish: {jobContext.Result ?? TaskResult.Succeeded}");
Trace.Info("Completing the job execution context.");
return(await CompleteJobAsync(jobServer, jobContext, message));
}
finally
{
if (runnerShutdownRegistration != null)
{
runnerShutdownRegistration.Value.Dispose();
runnerShutdownRegistration = null;
}
await ShutdownQueue(throwOnFailure : false);
}
}
private TestHostContext CreateTestContext([CallerMemberName] String testName = "")
{
var hc = new TestHostContext(this, testName);
_jobEc = new Runner.Worker.ExecutionContext();
_actionManager = new Mock <IActionManager>();
_jobServerQueue = new Mock <IJobServerQueue>();
_config = new Mock <IConfigurationStore>();
_logger = new Mock <IPagingLogger>();
_containerProvider = new Mock <IContainerOperationProvider>();
_diagnosticLogManager = new Mock <IDiagnosticLogManager>();
_directoryManager = new Mock <IPipelineDirectoryManager>();
_directoryManager.Setup(x => x.PrepareDirectory(It.IsAny <IExecutionContext>(), It.IsAny <Pipelines.WorkspaceOptions>()))
.Returns(new TrackingConfig()
{
PipelineDirectory = "runner", WorkspaceDirectory = "runner/runner"
});
IActionRunner step1 = new ActionRunner();
IActionRunner step2 = new ActionRunner();
IActionRunner step3 = new ActionRunner();
IActionRunner step4 = new ActionRunner();
IActionRunner step5 = new ActionRunner();
_logger.Setup(x => x.Setup(It.IsAny <Guid>(), It.IsAny <Guid>()));
var settings = new RunnerSettings
{
AgentId = 1,
AgentName = "runner",
ServerUrl = "https://pipelines.actions.githubusercontent.com/abcd",
WorkFolder = "_work",
};
_config.Setup(x => x.GetSettings())
.Returns(settings);
if (_tokenSource != null)
{
_tokenSource.Dispose();
_tokenSource = null;
}
_tokenSource = new CancellationTokenSource();
TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
TimelineReference timeline = new Timeline(Guid.NewGuid());
List <Pipelines.ActionStep> steps = new List <Pipelines.ActionStep>()
{
new Pipelines.ActionStep()
{
Id = Guid.NewGuid(),
DisplayName = "action1",
},
new Pipelines.ActionStep()
{
Id = Guid.NewGuid(),
DisplayName = "action2",
},
new Pipelines.ActionStep()
{
Id = Guid.NewGuid(),
DisplayName = "action3",
},
new Pipelines.ActionStep()
{
Id = Guid.NewGuid(),
DisplayName = "action4",
},
new Pipelines.ActionStep()
{
Id = Guid.NewGuid(),
DisplayName = "action5",
}
};
Guid jobId = Guid.NewGuid();
_message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary <string, VariableValue>(), new List <MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), steps, null, null, null, null);
GitHubContext github = new GitHubContext();
github["repository"] = new Pipelines.ContextData.StringContextData("actions/runner");
_message.ContextData.Add("github", github);
hc.SetSingleton(_actionManager.Object);
hc.SetSingleton(_config.Object);
hc.SetSingleton(_jobServerQueue.Object);
hc.SetSingleton(_containerProvider.Object);
hc.SetSingleton(_directoryManager.Object);
hc.SetSingleton(_diagnosticLogManager.Object);
hc.EnqueueInstance <IPagingLogger>(_logger.Object); // JobExecutionContext
hc.EnqueueInstance <IPagingLogger>(_logger.Object); // Initial Job
hc.EnqueueInstance <IPagingLogger>(_logger.Object); // step1
hc.EnqueueInstance <IPagingLogger>(_logger.Object); // step2
hc.EnqueueInstance <IPagingLogger>(_logger.Object); // step3
hc.EnqueueInstance <IPagingLogger>(_logger.Object); // step4
hc.EnqueueInstance <IPagingLogger>(_logger.Object); // step5
hc.EnqueueInstance <IPagingLogger>(_logger.Object); // prepare1
hc.EnqueueInstance <IPagingLogger>(_logger.Object); // prepare2
hc.EnqueueInstance <IActionRunner>(step1);
hc.EnqueueInstance <IActionRunner>(step2);
hc.EnqueueInstance <IActionRunner>(step3);
hc.EnqueueInstance <IActionRunner>(step4);
hc.EnqueueInstance <IActionRunner>(step5);
_jobEc.Initialize(hc);
_jobEc.InitializeJob(_message, _tokenSource.Token);
return(hc);
}
// Download all required actions.
// Make sure all condition inputs are valid.
// Build up three list of steps for jobrunner (pre-job, job, post-job).
public async Task <List <IStep> > InitializeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message)
{
Trace.Entering();
ArgUtil.NotNull(jobContext, nameof(jobContext));
ArgUtil.NotNull(message, nameof(message));
// Create a new timeline record for 'Set up job'
IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), "Set up job", $"{nameof(JobExtension)}_Init", null, null);
List <IStep> preJobSteps = new List <IStep>();
List <IStep> jobSteps = new List <IStep>();
using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
{
try
{
context.Start();
context.Debug($"Starting: Set up job");
context.Output($"Current runner version: '{BuildConstants.RunnerPackage.Version}'");
var repoFullName = context.GetGitHubContext("repository");
ArgUtil.NotNull(repoFullName, nameof(repoFullName));
context.Debug($"Primary repository: {repoFullName}");
// Print proxy setting information for better diagnostic experience
if (!string.IsNullOrEmpty(HostContext.WebProxy.HttpProxyAddress))
{
context.Output($"Runner is running behind proxy server '{HostContext.WebProxy.HttpProxyAddress}' for all HTTP requests.");
}
if (!string.IsNullOrEmpty(HostContext.WebProxy.HttpsProxyAddress))
{
context.Output($"Runner is running behind proxy server '{HostContext.WebProxy.HttpsProxyAddress}' for all HTTPS requests.");
}
// Prepare the workflow directory
context.Output("Prepare workflow directory");
var directoryManager = HostContext.GetService <IPipelineDirectoryManager>();
TrackingConfig trackingConfig = directoryManager.PrepareDirectory(
context,
message.Workspace);
// Set the directory variables
context.Debug("Update context data");
string _workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
context.SetRunnerContext("workspace", Path.Combine(_workDirectory, trackingConfig.PipelineDirectory));
context.SetGitHubContext("workspace", Path.Combine(_workDirectory, trackingConfig.WorkspaceDirectory));
// Evaluate the job-level environment variables
context.Debug("Evaluating job-level environment variables");
var templateEvaluator = context.ToPipelineTemplateEvaluator();
foreach (var token in message.EnvironmentVariables)
{
var environmentVariables = templateEvaluator.EvaluateStepEnvironment(token, jobContext.ExpressionValues, VarUtil.EnvironmentVariableKeyComparer);
foreach (var pair in environmentVariables)
{
context.EnvironmentVariables[pair.Key] = pair.Value ?? string.Empty;
context.SetEnvContext(pair.Key, pair.Value ?? string.Empty);
}
}
// Evaluate the job container
context.Debug("Evaluating job container");
var container = templateEvaluator.EvaluateJobContainer(message.JobContainer, jobContext.ExpressionValues);
if (container != null)
{
jobContext.Container = new Container.ContainerInfo(HostContext, container);
}
// Evaluate the job service containers
context.Debug("Evaluating job service containers");
var serviceContainers = templateEvaluator.EvaluateJobServiceContainers(message.JobServiceContainers, jobContext.ExpressionValues);
if (serviceContainers?.Count > 0)
{
foreach (var pair in serviceContainers)
{
var networkAlias = pair.Key;
var serviceContainer = pair.Value;
jobContext.ServiceContainers.Add(new Container.ContainerInfo(HostContext, serviceContainer, false, networkAlias));
}
}
// Build up 2 lists of steps, pre-job, job
// Download actions not already in the cache
Trace.Info("Downloading actions");
var actionManager = HostContext.GetService <IActionManager>();
var prepareSteps = await actionManager.PrepareActionsAsync(context, message.Steps);
preJobSteps.AddRange(prepareSteps);
// Add start-container steps, record and stop-container steps
if (jobContext.Container != null || jobContext.ServiceContainers.Count > 0)
{
var containerProvider = HostContext.GetService <IContainerOperationProvider>();
var containers = new List <Container.ContainerInfo>();
if (jobContext.Container != null)
{
containers.Add(jobContext.Container);
}
containers.AddRange(jobContext.ServiceContainers);
preJobSteps.Add(new JobExtensionRunner(runAsync: containerProvider.StartContainersAsync,
condition: $"{PipelineTemplateConstants.Success}()",
displayName: "Initialize containers",
data: (object)containers));
}
// Add action steps
foreach (var step in message.Steps)
{
if (step.Type == Pipelines.StepType.Action)
{
var action = step as Pipelines.ActionStep;
Trace.Info($"Adding {action.DisplayName}.");
var actionRunner = HostContext.CreateService <IActionRunner>();
actionRunner.Action = action;
actionRunner.Stage = ActionRunStage.Main;
actionRunner.Condition = step.Condition;
var contextData = new Pipelines.ContextData.DictionaryContextData();
if (message.ContextData?.Count > 0)
{
foreach (var pair in message.ContextData)
{
contextData[pair.Key] = pair.Value;
}
}
actionRunner.TryEvaluateDisplayName(contextData, context);
jobSteps.Add(actionRunner);
}
}
// Create execution context for pre-job steps
foreach (var step in preJobSteps)
{
if (step is JobExtensionRunner)
{
JobExtensionRunner extensionStep = step as JobExtensionRunner;
ArgUtil.NotNull(extensionStep, extensionStep.DisplayName);
Guid stepId = Guid.NewGuid();
extensionStep.ExecutionContext = jobContext.CreateChild(stepId, extensionStep.DisplayName, null, null, stepId.ToString("N"));
}
}
// Create execution context for job steps
foreach (var step in jobSteps)
{
if (step is IActionRunner actionStep)
{
ArgUtil.NotNull(actionStep, step.DisplayName);
actionStep.ExecutionContext = jobContext.CreateChild(actionStep.Action.Id, actionStep.DisplayName, actionStep.Action.Name, actionStep.Action.ScopeName, actionStep.Action.ContextName);
}
}
List <IStep> steps = new List <IStep>();
steps.AddRange(preJobSteps);
steps.AddRange(jobSteps);
// Prepare for orphan process cleanup
_processCleanup = jobContext.Variables.GetBoolean("process.clean") ?? true;
if (_processCleanup)
{
// Set the RUNNER_TRACKING_ID env variable.
Environment.SetEnvironmentVariable(Constants.ProcessTrackingId, _processLookupId);
context.Debug("Collect running processes for tracking orphan processes.");
// Take a snapshot of current running processes
Dictionary <int, Process> processes = SnapshotProcesses();
foreach (var proc in processes)
{
// Pid_ProcessName
_existingProcesses.Add($"{proc.Key}_{proc.Value.ProcessName}");
}
}
return(steps);
}
catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
{
// Log the exception and cancel the JobExtension Initialization.
Trace.Error($"Caught cancellation exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Canceled;
throw;
}
catch (Exception ex)
{
// Log the error and fail the JobExtension Initialization.
Trace.Error($"Caught exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Failed;
throw;
}
finally
{
context.Debug("Finishing: Set up job");
context.Complete();
}
}
}
public void FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, DateTime jobStartTimeUtc)
{
Trace.Entering();
ArgUtil.NotNull(jobContext, nameof(jobContext));
// create a new timeline record node for 'Finalize job'
IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), "Complete job", $"{nameof(JobExtension)}_Final", null, null);
using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
{
try
{
context.Start();
context.Debug("Starting: Complete job");
if (context.Variables.GetBoolean(Constants.Variables.Actions.RunnerDebug) ?? false)
{
Trace.Info("Support log upload starting.");
context.Output("Uploading runner diagnostic logs");
IDiagnosticLogManager diagnosticLogManager = HostContext.GetService <IDiagnosticLogManager>();
try
{
diagnosticLogManager.UploadDiagnosticLogs(executionContext: context, parentContext: jobContext, message: message, jobStartTimeUtc: jobStartTimeUtc);
Trace.Info("Support log upload complete.");
context.Output("Completed runner diagnostic log upload");
}
catch (Exception ex)
{
// Log the error but make sure we continue gracefully.
Trace.Info("Error uploading support logs.");
context.Output("Error uploading runner diagnostic logs");
Trace.Error(ex);
}
}
if (_processCleanup)
{
context.Output("Cleaning up orphan processes");
// Only check environment variable for any process that doesn't run before we invoke our process.
Dictionary <int, Process> currentProcesses = SnapshotProcesses();
foreach (var proc in currentProcesses)
{
if (proc.Key == Process.GetCurrentProcess().Id)
{
// skip for current process.
continue;
}
if (_existingProcesses.Contains($"{proc.Key}_{proc.Value.ProcessName}"))
{
Trace.Verbose($"Skip existing process. PID: {proc.Key} ({proc.Value.ProcessName})");
}
else
{
Trace.Info($"Inspecting process environment variables. PID: {proc.Key} ({proc.Value.ProcessName})");
string lookupId = null;
try
{
lookupId = proc.Value.GetEnvironmentVariable(HostContext, Constants.ProcessTrackingId);
}
catch (Exception ex)
{
Trace.Warning($"Ignore exception during read process environment variables: {ex.Message}");
Trace.Verbose(ex.ToString());
}
if (string.Equals(lookupId, _processLookupId, StringComparison.OrdinalIgnoreCase))
{
context.Output($"Terminate orphan process: pid ({proc.Key}) ({proc.Value.ProcessName})");
try
{
proc.Value.Kill();
}
catch (Exception ex)
{
Trace.Error("Catch exception during orphan process cleanup.");
Trace.Error(ex);
}
}
}
}
}
}
catch (Exception ex)
{
// Log and ignore the error from JobExtension finalization.
Trace.Error($"Caught exception from JobExtension finalization: {ex}");
context.Output(ex.Message);
}
finally
{
context.Debug("Finishing: Complete job");
context.Complete();
}
}
}
public void EchoProcessCommandDebugOn()
{
using (TestHostContext _hc = new TestHostContext(this))
{
// Set up a few things
// 1. Job request message (with ACTIONS_STEP_DEBUG = true)
TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
TimelineReference timeline = new TimelineReference();
Guid jobId = Guid.NewGuid();
string jobName = "some job name";
var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary <string, VariableValue>(), new List <MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List <Pipelines.ActionStep>(), null);
jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
{
Alias = Pipelines.PipelineConstants.SelfAlias,
Id = "github",
Version = "sha1"
});
jobRequest.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData();
jobRequest.Variables["ACTIONS_STEP_DEBUG"] = "true";
// Some service dependencies
var jobServerQueue = new Mock <IJobServerQueue>();
jobServerQueue.Setup(x => x.QueueTimelineRecordUpdate(It.IsAny <Guid>(), It.IsAny <TimelineRecord>()));
_hc.SetSingleton(jobServerQueue.Object);
var extensionManager = new Mock <IExtensionManager>();
var echoCommand = new EchoCommandExtension();
echoCommand.Initialize(_hc);
extensionManager.Setup(x => x.GetExtensions <IActionCommandExtension>())
.Returns(new List <IActionCommandExtension>()
{
echoCommand
});
_hc.SetSingleton <IExtensionManager>(extensionManager.Object);
var configurationStore = new Mock <IConfigurationStore>();
configurationStore.Setup(x => x.GetSettings()).Returns(new RunnerSettings());
_hc.SetSingleton(configurationStore.Object);
var pagingLogger = new Mock <IPagingLogger>();
_hc.EnqueueInstance(pagingLogger.Object);
ActionCommandManager commandManager = new ActionCommandManager();
commandManager.Initialize(_hc);
var _ec = new Runner.Worker.ExecutionContext();
_ec.Initialize(_hc);
// Initialize the job (to exercise logic that sets EchoOnActionCommand)
_ec.InitializeJob(jobRequest, System.Threading.CancellationToken.None);
_ec.Complete();
Assert.True(_ec.EchoOnActionCommand);
Assert.True(commandManager.TryProcessCommand(_ec, "::echo::off"));
Assert.False(_ec.EchoOnActionCommand);
Assert.True(commandManager.TryProcessCommand(_ec, "::echo::on"));
Assert.True(_ec.EchoOnActionCommand);
}
}
