public void Execute(IExecutionContext context, Command command)
{
var eventProperties = command.Properties;
var data = command.Data;
string resultText;
TaskResult result;
if (!eventProperties.TryGetValue(TaskCompleteEventProperties.Result, out resultText) ||
String.IsNullOrEmpty(resultText) ||
!Enum.TryParse <TaskResult>(resultText, out result))
{
throw new Exception(StringUtil.Loc("InvalidCommandResult"));
}
context.Result = TaskResultUtil.MergeTaskResults(context.Result, result);
context.Progress(100, data);
if (eventProperties.TryGetValue(TaskCompleteEventProperties.Done, out string doneText) &&
!String.IsNullOrEmpty(doneText) &&
StringUtil.ConvertToBoolean(doneText))
{
context.ForceTaskComplete();
}
}
private async Task <TestResults> RunWorker(HostContext HostContext, Pipelines.AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken)
{
var worker = HostContext.GetService <IWorker>();
Task <int> workerTask = null;
// Setup the anonymous pipes to use for communication with the worker.
using (var processChannel = HostContext.CreateService <IProcessChannel>())
{
processChannel.StartServer(startProcess: (string pipeHandleOut, string pipeHandleIn) =>
{
// Run the worker
// Note: this happens on the same process as the test
workerTask = worker.RunAsync(
pipeIn: pipeHandleOut,
pipeOut: pipeHandleIn);
}, disposeClient: false); // Don't dispose the client because our process is both the client and the server
// Send the job request message to the worker
var body = JsonUtility.ToString(message);
using (var csSendJobRequest = new CancellationTokenSource(ChannelTimeout))
{
await processChannel.SendAsync(
messageType : MessageType.NewJobRequest,
body : body,
cancellationToken : csSendJobRequest.Token);
}
// wait for worker process or cancellation token been fired.
var completedTask = await Task.WhenAny(workerTask, Task.Delay(-1, jobRequestCancellationToken));
if (completedTask == workerTask)
{
int returnCode = await workerTask;
TaskResult result = TaskResultUtil.TranslateFromReturnCode(returnCode);
return(new TestResults
{
ReturnCode = returnCode,
Result = result
});
}
else
{
return(new TestResults
{
TimedOut = true
});
}
}
}
private void ProcessTaskCompleteCommand(IExecutionContext context, Dictionary <string, string> eventProperties, String data)
{
string resultText;
TaskResult result;
if (!eventProperties.TryGetValue(TaskCompleteEventProperties.Result, out resultText) ||
String.IsNullOrEmpty(resultText) ||
!Enum.TryParse <TaskResult>(resultText, out result))
{
throw new Exception(StringUtil.Loc("InvalidCommandResult"));
}
context.Result = TaskResultUtil.MergeTaskResults(context.Result, result);
context.Progress(100, data);
}
public void TaskResultReturnCodeTranslate()
{
// Arrange.
using (TestHostContext hc = new TestHostContext(this))
{
// Act.
TaskResult abandon = TaskResultUtil.TranslateFromReturnCode(TaskResultUtil.TranslateToReturnCode(TaskResult.Abandoned));
// Actual
Assert.Equal(TaskResult.Abandoned, abandon);
// Act.
TaskResult canceled = TaskResultUtil.TranslateFromReturnCode(TaskResultUtil.TranslateToReturnCode(TaskResult.Canceled));
// Actual
Assert.Equal(TaskResult.Canceled, canceled);
// Act.
TaskResult failed = TaskResultUtil.TranslateFromReturnCode(TaskResultUtil.TranslateToReturnCode(TaskResult.Failed));
// Actual
Assert.Equal(TaskResult.Failed, failed);
// Act.
TaskResult skipped = TaskResultUtil.TranslateFromReturnCode(TaskResultUtil.TranslateToReturnCode(TaskResult.Skipped));
// Actual
Assert.Equal(TaskResult.Skipped, skipped);
// Act.
TaskResult succeeded = TaskResultUtil.TranslateFromReturnCode(TaskResultUtil.TranslateToReturnCode(TaskResult.Succeeded));
// Actual
Assert.Equal(TaskResult.Succeeded, succeeded);
// Act.
TaskResult succeededWithIssues = TaskResultUtil.TranslateFromReturnCode(TaskResultUtil.TranslateToReturnCode(TaskResult.SucceededWithIssues));
// Actual
Assert.Equal(TaskResult.SucceededWithIssues, succeededWithIssues);
// Act.
TaskResult unknowReturnCode1 = TaskResultUtil.TranslateFromReturnCode(0);
// Actual
Assert.Equal(TaskResult.Failed, unknowReturnCode1);
// Act.
TaskResult unknowReturnCode2 = TaskResultUtil.TranslateFromReturnCode(1);
// Actual
Assert.Equal(TaskResult.Failed, unknowReturnCode2);
}
}
private static void SetupResultCallback(string callbackId, Task taskResult)
{
taskResult.ContinueWith(task =>
{
if (task.Status == TaskStatus.RanToCompletion)
{
if (task.GetType() == typeof(Task))
{
RegisteredFunction.Invoke <bool>(
InvokePromiseCallback,
callbackId,
new InvocationResult <object> {
Succeeded = true, Result = null
});
}
else
{
var returnValue = TaskResultUtil.GetTaskResult(task);
RegisteredFunction.Invoke <bool>(
InvokePromiseCallback,
callbackId,
new InvocationResult <object> {
Succeeded = true, Result = returnValue
});
}
}
else
{
Exception exception = task.Exception;
while (exception is AggregateException || exception.InnerException is TargetInvocationException)
{
exception = exception.InnerException;
}
RegisteredFunction.Invoke <bool>(
InvokePromiseCallback,
callbackId,
new InvocationResult <object> {
Succeeded = false, Message = exception.Message
});
}
});
}
public async Task RunAsync(IExecutionContext jobContext, IList <IStep> steps)
{
// TaskResult:
// Abandoned
// Canceled
// Failed
// Skipped
// Succeeded
// SucceededWithIssues
bool stepFailed = false;
bool criticalStepFailed = false;
foreach (IStep step in steps)
{
Trace.Info($"Processing step: DisplayName='{step.DisplayName}', AlwaysRun={step.AlwaysRun}, ContinueOnError={step.ContinueOnError}, Critical={step.Critical}, Enabled={step.Enabled}, Finally={step.Finally}");
// TODO: Disabled steps may have already been removed. Investigate.
// Skip the current step if it is not Enabled.
if (!step.Enabled
// Or if a previous step failed and the current step is not AlwaysRun.
|| (stepFailed && !step.AlwaysRun && !step.Finally)
// Or if a previous Critical step failed and the current step is not Finally.
|| (criticalStepFailed && !step.Finally))
{
Trace.Info("Skipping step.");
step.ExecutionContext.Result = TaskResult.Skipped;
continue;
}
// Run the step.
Trace.Info("Starting the step.");
step.ExecutionContext.Start();
List <OperationCanceledException> allCancelExceptions = new List <OperationCanceledException>();
try
{
await step.RunAsync();
}
catch (OperationCanceledException ex)
{
// Log the exception and cancel the step.
Trace.Error($"Caught cancellation exception from step: {ex}");
step.ExecutionContext.Error(ex);
step.ExecutionContext.Result = TaskResult.Canceled;
//save the OperationCanceledException, merge with OperationCanceledException throw from Async Commands.
allCancelExceptions.Add(ex);
}
catch (Exception ex)
{
// Log the error and fail the step.
Trace.Error($"Caught exception from step: {ex}");
step.ExecutionContext.Error(ex);
step.ExecutionContext.Result = TaskResult.Failed;
}
// Wait till all async commands finish.
foreach (var command in step.ExecutionContext.AsyncCommands ?? new List <IAsyncCommandContext>())
{
try
{
// wait async command to finish.
await command.WaitAsync();
}
catch (OperationCanceledException ex)
{
// log and save the OperationCanceledException, set step result to canceled if the current result is not failed.
Trace.Error($"Caught cancellation exception from async command {command.Name}: {ex}");
step.ExecutionContext.Error(ex);
// if the step already failed, don't set it to canceled.
step.ExecutionContext.CommandResult = TaskResultUtil.MergeTaskResults(step.ExecutionContext.CommandResult, TaskResult.Canceled);
allCancelExceptions.Add(ex);
}
catch (Exception ex)
{
// Log the error, set step result to falied if the current result is not canceled.
Trace.Error($"Caught exception from async command {command.Name}: {ex}");
step.ExecutionContext.Error(ex);
// if the step already canceled, don't set it to failed.
step.ExecutionContext.CommandResult = TaskResultUtil.MergeTaskResults(step.ExecutionContext.CommandResult, TaskResult.Failed);
}
}
// Merge executioncontext result with command result
if (step.ExecutionContext.CommandResult != null)
{
step.ExecutionContext.Result = TaskResultUtil.MergeTaskResults(step.ExecutionContext.Result, step.ExecutionContext.CommandResult.Value);
}
// TODO: consider use token.IsCancellationRequest determine step cancelled instead of catch OperationCancelException all over the place
if (step.ExecutionContext.Result == TaskResult.Canceled && allCancelExceptions.Count > 0)
{
step.ExecutionContext.Complete();
throw allCancelExceptions.First();
}
// Fixup the step result if ContinueOnError.
if (step.ExecutionContext.Result == TaskResult.Failed && step.ContinueOnError)
{
step.ExecutionContext.Result = TaskResult.SucceededWithIssues;
Trace.Info($"Updated step result: {step.ExecutionContext.Result}");
}
else
{
Trace.Info($"Step result: {step.ExecutionContext.Result}");
}
// Complete the step context.
step.ExecutionContext.Complete();
// Update the step failed flags.
stepFailed = stepFailed || step.ExecutionContext.Result == TaskResult.Failed;
criticalStepFailed = criticalStepFailed || (step.Critical && step.ExecutionContext.Result == TaskResult.Failed);
// Update the job result.
if (step.ExecutionContext.Result == TaskResult.Failed)
{
jobContext.Result = TaskResult.Failed;
}
else if ((jobContext.Result ?? TaskResult.Succeeded) == TaskResult.Succeeded &&
step.ExecutionContext.Result == TaskResult.SucceededWithIssues)
{
jobContext.Result = TaskResult.SucceededWithIssues;
}
Trace.Info($"Current state: job state = '{jobContext.Result}', step failed = {stepFailed}, critical step failed = {criticalStepFailed}");
}
}
private async Task <TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, TaskResult?taskResult = null)
{
jobContext.Debug($"Finishing: {message.JobDisplayName}");
TaskResult result = jobContext.Complete(taskResult);
try
{
await ShutdownQueue(throwOnFailure : true);
}
catch (Exception ex)
{
Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(_jobServerQueue.ShutdownAsync)}");
Trace.Error("This indicate a failure during publish output variables. Fail the job to prevent unexpected job outputs.");
Trace.Error(ex);
result = TaskResultUtil.MergeTaskResults(result, TaskResult.Failed);
}
// Clean TEMP after finish process jobserverqueue, since there might be a pending fileupload still use the TEMP dir.
_tempDirectoryManager?.CleanupTempDirectory();
if (!jobContext.Global.Features.HasFlag(PlanFeatures.JobCompletedPlanEvent))
{
Trace.Info($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}");
return(result);
}
// Load any upgrade telemetry
LoadFromTelemetryFile(jobContext.JobTelemetry);
// Make sure we don't submit secrets as telemetry
MaskTelemetrySecrets(jobContext.JobTelemetry);
Trace.Info("Raising job completed event.");
var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result, jobContext.JobOutputs, jobContext.ActionsEnvironment, jobContext.ActionsStepsTelemetry, jobContext.JobTelemetry);
var completeJobRetryLimit = 5;
var exceptions = new List <Exception>();
while (completeJobRetryLimit-- > 0)
{
try
{
await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, default(CancellationToken));
return(result);
}
catch (TaskOrchestrationPlanNotFoundException ex)
{
Trace.Error($"TaskOrchestrationPlanNotFoundException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
Trace.Error(ex);
return(TaskResult.Failed);
}
catch (TaskOrchestrationPlanSecurityException ex)
{
Trace.Error($"TaskOrchestrationPlanSecurityException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
Trace.Error(ex);
return(TaskResult.Failed);
}
catch (TaskOrchestrationPlanTerminatedException ex)
{
Trace.Error($"TaskOrchestrationPlanTerminatedException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
Trace.Error(ex);
return(TaskResult.Failed);
}
catch (Exception ex)
{
Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}.");
Trace.Error(ex);
exceptions.Add(ex);
}
// delay 5 seconds before next retry.
await Task.Delay(TimeSpan.FromSeconds(5));
}
// rethrow exceptions from all attempts.
throw new AggregateException(exceptions);
}
private async Task RunAsync(Pipelines.AgentJobRequestMessage message, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken, CancellationToken workerCancelTimeoutKillToken)
{
if (previousJobDispatch != null)
{
Trace.Verbose($"Make sure the previous job request {previousJobDispatch.JobId} has successfully finished on worker.");
await EnsureDispatchFinished(previousJobDispatch);
}
else
{
Trace.Verbose($"This is the first job request.");
}
var term = HostContext.GetService <ITerminal>();
term.WriteLine(StringUtil.Loc("RunningJob", DateTime.UtcNow, message.JobDisplayName));
// first job request renew succeed.
TaskCompletionSource <int> firstJobRequestRenewed = new TaskCompletionSource <int>();
var notification = HostContext.GetService <IJobNotification>();
// lock renew cancellation token.
using (var lockRenewalTokenSource = new CancellationTokenSource())
using (var workerProcessCancelTokenSource = new CancellationTokenSource())
{
long requestId = message.RequestId;
Guid lockToken = Guid.Empty; // lockToken has never been used, keep this here of compat
// start renew job request
Trace.Info($"Start renew job request {requestId} for job {message.JobId}.");
Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, firstJobRequestRenewed, lockRenewalTokenSource.Token);
// wait till first renew succeed or job request is canceled
// not even start worker if the first renew fail
await Task.WhenAny(firstJobRequestRenewed.Task, renewJobRequest, Task.Delay(-1, jobRequestCancellationToken));
if (renewJobRequest.IsCompleted)
{
// renew job request task complete means we run out of retry for the first job request renew.
Trace.Info($"Unable to renew job request for job {message.JobId} for the first time, stop dispatching job to worker.");
return;
}
if (jobRequestCancellationToken.IsCancellationRequested)
{
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request with result Cancelled
await CompleteJobRequestAsync(_poolId, message, lockToken, TaskResult.Canceled);
return;
}
HostContext.WritePerfCounter($"JobRequestRenewed_{requestId.ToString()}");
Task <int> workerProcessTask = null;
object _outputLock = new object();
List <string> workerOutput = new List <string>();
using (var processChannel = HostContext.CreateService <IProcessChannel>())
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
// Start the process channel.
// It's OK if StartServer bubbles an execption after the worker process has already started.
// The worker will shutdown after 30 seconds if it hasn't received the job message.
processChannel.StartServer(
// Delegate to start the child process.
startProcess: (string pipeHandleOut, string pipeHandleIn) =>
{
// Validate args.
ArgUtil.NotNullOrEmpty(pipeHandleOut, nameof(pipeHandleOut));
ArgUtil.NotNullOrEmpty(pipeHandleIn, nameof(pipeHandleIn));
if (HostContext.RunMode == RunMode.Normal)
{
// Save STDOUT from worker, worker will use STDOUT report unhandle exception.
processInvoker.OutputDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stdout)
{
if (!string.IsNullOrEmpty(stdout.Data))
{
lock (_outputLock)
{
workerOutput.Add(stdout.Data);
}
}
};
// Save STDERR from worker, worker will use STDERR on crash.
processInvoker.ErrorDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stderr)
{
if (!string.IsNullOrEmpty(stderr.Data))
{
lock (_outputLock)
{
workerOutput.Add(stderr.Data);
}
}
};
}
else if (HostContext.RunMode == RunMode.Local)
{
processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) => Console.WriteLine(e.Data);
processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) => Console.WriteLine(e.Data);
}
// Start the child process.
HostContext.WritePerfCounter("StartingWorkerProcess");
var assemblyDirectory = HostContext.GetDirectory(WellKnownDirectory.Bin);
string workerFileName = Path.Combine(assemblyDirectory, _workerProcessName);
workerProcessTask = processInvoker.ExecuteAsync(
workingDirectory: assemblyDirectory,
fileName: workerFileName,
arguments: "spawnclient " + pipeHandleOut + " " + pipeHandleIn,
environment: null,
requireExitCodeZero: false,
outputEncoding: null,
killProcessOnCancel: true,
cancellationToken: workerProcessCancelTokenSource.Token);
});
// Send the job request message.
// Kill the worker process if sending the job message times out. The worker
// process may have successfully received the job message.
try
{
Trace.Info($"Send job request message to worker for job {message.JobId}.");
HostContext.WritePerfCounter($"AgentSendingJobToWorker_{message.JobId}");
using (var csSendJobRequest = new CancellationTokenSource(_channelTimeout))
{
await processChannel.SendAsync(
messageType : MessageType.NewJobRequest,
body : JsonUtility.ToString(message),
cancellationToken : csSendJobRequest.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
// timeout 30 sec. kill worker.
Trace.Info($"Job request message sending for job {message.JobId} been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// not finish the job request since the job haven't run on worker at all, we will not going to set a result to server.
return;
}
// we get first jobrequest renew succeed and start the worker process with the job message.
// send notification to machine provisioner.
await notification.JobStarted(message.JobId);
HostContext.WritePerfCounter($"SentJobToWorker_{requestId.ToString()}");
try
{
TaskResult resultOnAbandonOrCancel = TaskResult.Succeeded;
// wait for renewlock, worker process or cancellation token been fired.
var completedTask = await Task.WhenAny(renewJobRequest, workerProcessTask, Task.Delay(-1, jobRequestCancellationToken));
if (completedTask == workerProcessTask)
{
// worker finished successfully, complete job request with result, attach unhandled exception reported by worker, stop renew lock, job has finished.
int returnCode = await workerProcessTask;
Trace.Info($"Worker finished for job {message.JobId}. Code: " + returnCode);
string detailInfo = null;
if (!TaskResultUtil.IsValidReturnCode(returnCode))
{
detailInfo = string.Join(Environment.NewLine, workerOutput);
Trace.Info($"Return code {returnCode} indicate worker encounter an unhandled exception or app crash, attach worker stdout/stderr to JobRequest result.");
await LogWorkerProcessUnhandledException(message, detailInfo);
}
TaskResult result = TaskResultUtil.TranslateFromReturnCode(returnCode);
Trace.Info($"finish job request for job {message.JobId} with result: {result}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobDisplayName, result));
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request
await CompleteJobRequestAsync(_poolId, message, lockToken, result, detailInfo);
// print out unhandled exception happened in worker after we complete job request.
// when we run out of disk space, report back to server has higher priority.
if (!string.IsNullOrEmpty(detailInfo))
{
Trace.Error("Unhandled exception happened in worker:");
Trace.Error(detailInfo);
}
return;
}
else if (completedTask == renewJobRequest)
{
resultOnAbandonOrCancel = TaskResult.Abandoned;
}
else
{
resultOnAbandonOrCancel = TaskResult.Canceled;
}
// renew job request completed or job request cancellation token been fired for RunAsync(jobrequestmessage)
// cancel worker gracefully first, then kill it after worker cancel timeout
try
{
Trace.Info($"Send job cancellation message to worker for job {message.JobId}.");
using (var csSendCancel = new CancellationTokenSource(_channelTimeout))
{
var messageType = MessageType.CancelRequest;
if (HostContext.AgentShutdownToken.IsCancellationRequested)
{
switch (HostContext.AgentShutdownReason)
{
case ShutdownReason.UserCancelled:
messageType = MessageType.AgentShutdown;
break;
case ShutdownReason.OperatingSystemShutdown:
messageType = MessageType.OperatingSystemShutdown;
break;
}
}
await processChannel.SendAsync(
messageType : messageType,
body : string.Empty,
cancellationToken : csSendCancel.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
Trace.Info($"Job cancel message sending for job {message.JobId} been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
}
// wait worker to exit
// if worker doesn't exit within timeout, then kill worker.
completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, workerCancelTimeoutKillToken));
// worker haven't exit within cancellation timeout.
if (completedTask != workerProcessTask)
{
Trace.Info($"worker process for job {message.JobId} haven't exit within cancellation timout, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
}
Trace.Info($"finish job request for job {message.JobId} with result: {resultOnAbandonOrCancel}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobDisplayName, resultOnAbandonOrCancel));
// complete job request with cancel result, stop renew lock, job has finished.
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request
await CompleteJobRequestAsync(_poolId, message, lockToken, resultOnAbandonOrCancel);
}
finally
{
// This should be the last thing to run so we don't notify external parties until actually finished
await notification.JobCompleted(message.JobId);
}
}
}
}
public async Task RunAsync(IExecutionContext jobContext, IList <IStep> steps)
{
ArgUtil.NotNull(jobContext, nameof(jobContext));
ArgUtil.NotNull(steps, nameof(steps));
// TaskResult:
// Abandoned
// Canceled
// Failed
// Skipped
// Succeeded
// SucceededWithIssues
bool stepFailed = false;
bool criticalStepFailed = false;
int stepCount = 0;
jobContext.Variables.Agent_JobStatus = TaskResult.Succeeded;
foreach (IStep step in steps)
{
Trace.Info($"Processing step: DisplayName='{step.DisplayName}', AlwaysRun={step.AlwaysRun}, ContinueOnError={step.ContinueOnError}, Critical={step.Critical}, Enabled={step.Enabled}, Finally={step.Finally}");
ArgUtil.Equal(true, step.Enabled, nameof(step.Enabled));
ArgUtil.NotNull(step.ExecutionContext, nameof(step.ExecutionContext));
ArgUtil.NotNull(step.ExecutionContext.Variables, nameof(step.ExecutionContext.Variables));
jobContext.Progress(stepCount++ *100 / steps.Count);
// TODO: Run finally even if canceled?
// Skip if a previous step failed and the current step is not AlwaysRun.
if ((stepFailed && !step.AlwaysRun && !step.Finally)
// Or if a previous Critical step failed and the current step is not Finally.
|| (criticalStepFailed && !step.Finally))
{
Trace.Info("Skipping step.");
step.ExecutionContext.Result = TaskResult.Skipped;
continue;
}
// Run the step.
Trace.Info("Starting the step.");
TimeSpan?stepTimeout = null;
if (step is ITaskRunner && (step as ITaskRunner).TaskInstance.TimeoutInMinutes > 0)
{
stepTimeout = TimeSpan.FromMinutes((step as ITaskRunner).TaskInstance.TimeoutInMinutes);
}
step.ExecutionContext.Start(timeout: stepTimeout);
List <string> expansionWarnings;
step.ExecutionContext.Variables.RecalculateExpanded(out expansionWarnings);
expansionWarnings?.ForEach(x => step.ExecutionContext.Warning(x));
List <OperationCanceledException> allCancelExceptions = new List <OperationCanceledException>();
try
{
await step.RunAsync();
}
catch (OperationCanceledException ex)
{
if (step.ExecutionContext.CancellationToken.IsCancellationRequested &&
!jobContext.CancellationToken.IsCancellationRequested)
{
Trace.Error($"Caught timeout exception from step: {ex.Message}");
step.ExecutionContext.Error(StringUtil.Loc("StepTimedOut"));
step.ExecutionContext.Result = TaskResult.Failed;
}
else
{
// Log the exception and cancel the step.
Trace.Error($"Caught cancellation exception from step: {ex}");
step.ExecutionContext.Error(ex);
step.ExecutionContext.Result = TaskResult.Canceled;
}
//save the OperationCanceledException, merge with OperationCanceledException throw from Async Commands.
allCancelExceptions.Add(ex);
}
catch (Exception ex)
{
// Log the error and fail the step.
Trace.Error($"Caught exception from step: {ex}");
step.ExecutionContext.Error(ex);
step.ExecutionContext.Result = TaskResult.Failed;
}
// Wait till all async commands finish.
foreach (var command in step.ExecutionContext.AsyncCommands ?? new List <IAsyncCommandContext>())
{
try
{
// wait async command to finish.
await command.WaitAsync();
}
catch (OperationCanceledException ex)
{
if (step.ExecutionContext.CancellationToken.IsCancellationRequested &&
!jobContext.CancellationToken.IsCancellationRequested)
{
// Log the timeout error, set step result to falied if the current result is not canceled.
Trace.Error($"Caught timeout exception from async command {command.Name}: {ex}");
step.ExecutionContext.Error(StringUtil.Loc("StepTimedOut"));
// if the step already canceled, don't set it to failed.
step.ExecutionContext.CommandResult = TaskResultUtil.MergeTaskResults(step.ExecutionContext.CommandResult, TaskResult.Failed);
}
else
{
// log and save the OperationCanceledException, set step result to canceled if the current result is not failed.
Trace.Error($"Caught cancellation exception from async command {command.Name}: {ex}");
step.ExecutionContext.Error(ex);
// if the step already failed, don't set it to canceled.
step.ExecutionContext.CommandResult = TaskResultUtil.MergeTaskResults(step.ExecutionContext.CommandResult, TaskResult.Canceled);
}
allCancelExceptions.Add(ex);
}
catch (Exception ex)
{
// Log the error, set step result to falied if the current result is not canceled.
Trace.Error($"Caught exception from async command {command.Name}: {ex}");
step.ExecutionContext.Error(ex);
// if the step already canceled, don't set it to failed.
step.ExecutionContext.CommandResult = TaskResultUtil.MergeTaskResults(step.ExecutionContext.CommandResult, TaskResult.Failed);
}
}
// Merge executioncontext result with command result
if (step.ExecutionContext.CommandResult != null)
{
step.ExecutionContext.Result = TaskResultUtil.MergeTaskResults(step.ExecutionContext.Result, step.ExecutionContext.CommandResult.Value);
}
// TODO: consider use token.IsCancellationRequest determine step cancelled instead of catch OperationCancelException all over the place
if (step.ExecutionContext.Result == TaskResult.Canceled && allCancelExceptions.Count > 0)
{
step.ExecutionContext.Complete();
throw allCancelExceptions.First();
}
// Fixup the step result if ContinueOnError.
if (step.ExecutionContext.Result == TaskResult.Failed && step.ContinueOnError)
{
step.ExecutionContext.Result = TaskResult.SucceededWithIssues;
Trace.Info($"Updated step result: {step.ExecutionContext.Result}");
}
else
{
Trace.Info($"Step result: {step.ExecutionContext.Result}");
}
// Complete the step context.
step.ExecutionContext.Complete();
// Update the step failed flags.
stepFailed = stepFailed || step.ExecutionContext.Result == TaskResult.Failed;
criticalStepFailed = criticalStepFailed || (step.Critical && step.ExecutionContext.Result == TaskResult.Failed);
// Update the job result.
if (step.ExecutionContext.Result == TaskResult.Failed)
{
jobContext.Result = TaskResult.Failed;
jobContext.Variables.Agent_JobStatus = TaskResult.Failed;
}
else if ((jobContext.Result ?? TaskResult.Succeeded) == TaskResult.Succeeded &&
step.ExecutionContext.Result == TaskResult.SucceededWithIssues)
{
jobContext.Result = TaskResult.SucceededWithIssues;
jobContext.Variables.Agent_JobStatus = TaskResult.SucceededWithIssues;
}
Trace.Info($"Current state: job state = '{jobContext.Result}', step failed = {stepFailed}, critical step failed = {criticalStepFailed}");
}
}
// StepsRunner should never throw exception to caller
public async Task RunAsync(IExecutionContext jobContext, IList <IStep> steps)
{
ArgUtil.NotNull(jobContext, nameof(jobContext));
ArgUtil.NotNull(steps, nameof(steps));
// TaskResult:
// Abandoned (Server set this.)
// Canceled
// Failed
// Skipped
// Succeeded
// SucceededWithIssues
CancellationTokenRegistration?jobCancelRegister = null;
int stepIndex = 0;
jobContext.Variables.Agent_JobStatus = jobContext.Result ?? TaskResult.Succeeded;
foreach (IStep step in steps)
{
Trace.Info($"Processing step: DisplayName='{step.DisplayName}', ContinueOnError={step.ContinueOnError}, Enabled={step.Enabled}");
ArgUtil.Equal(true, step.Enabled, nameof(step.Enabled));
ArgUtil.NotNull(step.ExecutionContext, nameof(step.ExecutionContext));
ArgUtil.NotNull(step.ExecutionContext.Variables, nameof(step.ExecutionContext.Variables));
stepIndex++;
// Start.
step.ExecutionContext.Start();
var taskStep = step as ITaskRunner;
if (taskStep != null)
{
HostContext.WritePerfCounter($"TaskStart_{taskStep.Task.Reference.Name}_{stepIndex}");
}
// Variable expansion.
List <string> expansionWarnings;
step.ExecutionContext.Variables.RecalculateExpanded(out expansionWarnings);
expansionWarnings?.ForEach(x => step.ExecutionContext.Warning(x));
var expressionManager = HostContext.GetService <IExpressionManager>();
try
{
// Register job cancellation call back only if job cancellation token not been fire before each step run
if (!jobContext.CancellationToken.IsCancellationRequested)
{
// Test the condition again. The job was canceled after the condition was originally evaluated.
jobCancelRegister = jobContext.CancellationToken.Register(() =>
{
// mark job as cancelled
jobContext.Result = TaskResult.Canceled;
jobContext.Variables.Agent_JobStatus = jobContext.Result;
step.ExecutionContext.Debug($"Re-evaluate condition on job cancellation for step: '{step.DisplayName}'.");
ConditionResult conditionReTestResult;
if (HostContext.AgentShutdownToken.IsCancellationRequested)
{
step.ExecutionContext.Debug($"Skip Re-evaluate condition on agent shutdown.");
conditionReTestResult = false;
}
else
{
try
{
conditionReTestResult = expressionManager.Evaluate(step.ExecutionContext, step.Condition, hostTracingOnly: true);
}
catch (Exception ex)
{
// Cancel the step since we get exception while re-evaluate step condition.
Trace.Info("Caught exception from expression when re-test condition on job cancellation.");
step.ExecutionContext.Error(ex);
conditionReTestResult = false;
}
}
if (!conditionReTestResult.Value)
{
// Cancel the step.
Trace.Info("Cancel current running step.");
step.ExecutionContext.CancelToken();
}
});
}
else
{
if (jobContext.Result != TaskResult.Canceled)
{
// mark job as cancelled
jobContext.Result = TaskResult.Canceled;
jobContext.Variables.Agent_JobStatus = jobContext.Result;
}
}
// Evaluate condition.
step.ExecutionContext.Debug($"Evaluating condition for step: '{step.DisplayName}'");
Exception conditionEvaluateError = null;
ConditionResult conditionResult;
if (HostContext.AgentShutdownToken.IsCancellationRequested)
{
step.ExecutionContext.Debug($"Skip evaluate condition on agent shutdown.");
conditionResult = false;
}
else
{
try
{
conditionResult = expressionManager.Evaluate(step.ExecutionContext, step.Condition);
}
catch (Exception ex)
{
Trace.Info("Caught exception from expression.");
Trace.Error(ex);
conditionResult = false;
conditionEvaluateError = ex;
}
}
// no evaluate error but condition is false
if (!conditionResult.Value && conditionEvaluateError == null)
{
// Condition == false
Trace.Info("Skipping step due to condition evaluation.");
step.ExecutionContext.Complete(TaskResult.Skipped, resultCode: conditionResult.Trace);
continue;
}
if (conditionEvaluateError != null)
{
// fail the step since there is an evaluate error.
step.ExecutionContext.Error(conditionEvaluateError);
step.ExecutionContext.Complete(TaskResult.Failed);
}
else
{
// Run the step.
await RunStepAsync(step, jobContext.CancellationToken);
}
}
finally
{
if (jobCancelRegister != null)
{
jobCancelRegister?.Dispose();
jobCancelRegister = null;
}
}
// Update the job result.
if (step.ExecutionContext.Result == TaskResult.SucceededWithIssues ||
step.ExecutionContext.Result == TaskResult.Failed)
{
Trace.Info($"Update job result with current step result '{step.ExecutionContext.Result}'.");
jobContext.Result = TaskResultUtil.MergeTaskResults(jobContext.Result, step.ExecutionContext.Result.Value);
jobContext.Variables.Agent_JobStatus = jobContext.Result;
}
else
{
Trace.Info($"No need for updating job result with current step result '{step.ExecutionContext.Result}'.");
}
if (taskStep != null)
{
HostContext.WritePerfCounter($"TaskCompleted_{taskStep.Task.Reference.Name}_{stepIndex}");
}
Trace.Info($"Current state: job state = '{jobContext.Result}'");
}
}
public void TaskResultsMerge()
{
// Arrange.
using (TestHostContext hc = new TestHostContext(this))
{
TaskResult merged;
//
// No current result merge.
//
// Act.
merged = TaskResultUtil.MergeTaskResults(null, TaskResult.Succeeded);
// Actual
Assert.Equal(TaskResult.Succeeded, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(null, TaskResult.SucceededWithIssues);
// Actual
Assert.Equal(TaskResult.SucceededWithIssues, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(null, TaskResult.Abandoned);
// Actual
Assert.Equal(TaskResult.Abandoned, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(null, TaskResult.Canceled);
// Actual
Assert.Equal(TaskResult.Canceled, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(null, TaskResult.Failed);
// Actual
Assert.Equal(TaskResult.Failed, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(null, TaskResult.Skipped);
// Actual
Assert.Equal(TaskResult.Skipped, merged);
//
// Same result merge.
//
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Succeeded, TaskResult.Succeeded);
// Actual
Assert.Equal(TaskResult.Succeeded, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.SucceededWithIssues, TaskResult.SucceededWithIssues);
// Actual
Assert.Equal(TaskResult.SucceededWithIssues, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Abandoned, TaskResult.Abandoned);
// Actual
Assert.Equal(TaskResult.Abandoned, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Canceled, TaskResult.Canceled);
// Actual
Assert.Equal(TaskResult.Canceled, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Failed, TaskResult.Failed);
// Actual
Assert.Equal(TaskResult.Failed, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Skipped, TaskResult.Skipped);
// Actual
Assert.Equal(TaskResult.Skipped, merged);
//
// Forward result merge
//
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Succeeded, TaskResult.SucceededWithIssues);
// Actual
Assert.Equal(TaskResult.SucceededWithIssues, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Succeeded, TaskResult.Abandoned);
// Actual
Assert.Equal(TaskResult.Abandoned, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Succeeded, TaskResult.Canceled);
// Actual
Assert.Equal(TaskResult.Canceled, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Succeeded, TaskResult.Failed);
// Actual
Assert.Equal(TaskResult.Failed, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Succeeded, TaskResult.Skipped);
// Actual
Assert.Equal(TaskResult.Skipped, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.SucceededWithIssues, TaskResult.Abandoned);
// Actual
Assert.Equal(TaskResult.Abandoned, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.SucceededWithIssues, TaskResult.Canceled);
// Actual
Assert.Equal(TaskResult.Canceled, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.SucceededWithIssues, TaskResult.Failed);
// Actual
Assert.Equal(TaskResult.Failed, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.SucceededWithIssues, TaskResult.Skipped);
// Actual
Assert.Equal(TaskResult.Skipped, merged);
//
// No backward merge
//
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Abandoned, TaskResult.SucceededWithIssues);
// Actual
Assert.Equal(TaskResult.Abandoned, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Abandoned, TaskResult.Succeeded);
// Actual
Assert.Equal(TaskResult.Abandoned, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Canceled, TaskResult.SucceededWithIssues);
// Actual
Assert.Equal(TaskResult.Canceled, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Canceled, TaskResult.Succeeded);
// Actual
Assert.Equal(TaskResult.Canceled, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Failed, TaskResult.SucceededWithIssues);
// Actual
Assert.Equal(TaskResult.Failed, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Failed, TaskResult.Succeeded);
// Actual
Assert.Equal(TaskResult.Failed, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Skipped, TaskResult.SucceededWithIssues);
// Actual
Assert.Equal(TaskResult.Skipped, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Skipped, TaskResult.Succeeded);
// Actual
Assert.Equal(TaskResult.Skipped, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.SucceededWithIssues, TaskResult.Succeeded);
// Actual
Assert.Equal(TaskResult.SucceededWithIssues, merged);
//
// Worst result no change
//
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Abandoned, TaskResult.Canceled);
// Actual
Assert.Equal(TaskResult.Abandoned, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Abandoned, TaskResult.Failed);
// Actual
Assert.Equal(TaskResult.Abandoned, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Abandoned, TaskResult.Skipped);
// Actual
Assert.Equal(TaskResult.Abandoned, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Canceled, TaskResult.Abandoned);
// Actual
Assert.Equal(TaskResult.Canceled, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Canceled, TaskResult.Failed);
// Actual
Assert.Equal(TaskResult.Canceled, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Canceled, TaskResult.Skipped);
// Actual
Assert.Equal(TaskResult.Canceled, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Failed, TaskResult.Abandoned);
// Actual
Assert.Equal(TaskResult.Abandoned, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Failed, TaskResult.Canceled);
// Actual
Assert.Equal(TaskResult.Canceled, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Failed, TaskResult.Skipped);
// Actual
Assert.Equal(TaskResult.Skipped, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Skipped, TaskResult.Abandoned);
// Actual
Assert.Equal(TaskResult.Skipped, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Skipped, TaskResult.Canceled);
// Actual
Assert.Equal(TaskResult.Skipped, merged);
// Act.
merged = TaskResultUtil.MergeTaskResults(TaskResult.Skipped, TaskResult.Failed);
// Actual
Assert.Equal(TaskResult.Skipped, merged);
}
}
private async Task RunStepAsync(IStep step, CancellationToken jobCancellationToken)
{
// Start the step.
Trace.Info("Starting the step.");
step.ExecutionContext.Section(StringUtil.Loc("StepStarting", step.DisplayName));
step.ExecutionContext.SetTimeout(timeout: step.Timeout);
#if OS_WINDOWS
try
{
if (step.ExecutionContext.Variables.Retain_Default_Encoding != true && Console.InputEncoding.CodePage != 65001)
{
using (var p = HostContext.CreateService <IProcessInvoker>())
{
// Use UTF8 code page
int exitCode = await p.ExecuteAsync(workingDirectory : HostContext.GetDirectory(WellKnownDirectory.Work),
fileName : WhichUtil.Which("chcp", true, Trace),
arguments : "65001",
environment : null,
requireExitCodeZero : false,
outputEncoding : null,
killProcessOnCancel : false,
redirectStandardIn : null,
inheritConsoleHandler : true,
cancellationToken : step.ExecutionContext.CancellationToken);
if (exitCode == 0)
{
Trace.Info("Successfully returned to code page 65001 (UTF8)");
}
else
{
Trace.Warning($"'chcp 65001' failed with exit code {exitCode}");
}
}
}
}
catch (Exception ex)
{
Trace.Warning($"'chcp 65001' failed with exception {ex.Message}");
}
#endif
try
{
await step.RunAsync();
}
catch (OperationCanceledException ex)
{
if (step.ExecutionContext.CancellationToken.IsCancellationRequested &&
!jobCancellationToken.IsCancellationRequested)
{
Trace.Error($"Caught timeout exception from step: {ex.Message}");
step.ExecutionContext.Error(StringUtil.Loc("StepTimedOut"));
step.ExecutionContext.Result = TaskResult.Failed;
}
else
{
// Log the exception and cancel the step.
Trace.Error($"Caught cancellation exception from step: {ex}");
step.ExecutionContext.Error(ex);
step.ExecutionContext.Result = TaskResult.Canceled;
}
}
catch (Exception ex)
{
// Log the error and fail the step.
Trace.Error($"Caught exception from step: {ex}");
step.ExecutionContext.Error(ex);
step.ExecutionContext.Result = TaskResult.Failed;
}
// Wait till all async commands finish.
foreach (var command in step.ExecutionContext.AsyncCommands ?? new List <IAsyncCommandContext>())
{
try
{
// wait async command to finish.
await command.WaitAsync();
}
catch (OperationCanceledException ex)
{
if (step.ExecutionContext.CancellationToken.IsCancellationRequested &&
!jobCancellationToken.IsCancellationRequested)
{
// Log the timeout error, set step result to falied if the current result is not canceled.
Trace.Error($"Caught timeout exception from async command {command.Name}: {ex}");
step.ExecutionContext.Error(StringUtil.Loc("StepTimedOut"));
// if the step already canceled, don't set it to failed.
step.ExecutionContext.CommandResult = TaskResultUtil.MergeTaskResults(step.ExecutionContext.CommandResult, TaskResult.Failed);
}
else
{
// log and save the OperationCanceledException, set step result to canceled if the current result is not failed.
Trace.Error($"Caught cancellation exception from async command {command.Name}: {ex}");
step.ExecutionContext.Error(ex);
// if the step already failed, don't set it to canceled.
step.ExecutionContext.CommandResult = TaskResultUtil.MergeTaskResults(step.ExecutionContext.CommandResult, TaskResult.Canceled);
}
}
catch (Exception ex)
{
// Log the error, set step result to falied if the current result is not canceled.
Trace.Error($"Caught exception from async command {command.Name}: {ex}");
step.ExecutionContext.Error(ex);
// if the step already canceled, don't set it to failed.
step.ExecutionContext.CommandResult = TaskResultUtil.MergeTaskResults(step.ExecutionContext.CommandResult, TaskResult.Failed);
}
}
// Merge executioncontext result with command result
if (step.ExecutionContext.CommandResult != null)
{
step.ExecutionContext.Result = TaskResultUtil.MergeTaskResults(step.ExecutionContext.Result, step.ExecutionContext.CommandResult.Value);
}
// Fixup the step result if ContinueOnError.
if (step.ExecutionContext.Result == TaskResult.Failed && step.ContinueOnError)
{
step.ExecutionContext.Result = TaskResult.SucceededWithIssues;
Trace.Info($"Updated step result: {step.ExecutionContext.Result}");
}
else
{
Trace.Info($"Step result: {step.ExecutionContext.Result}");
}
// Complete the step context.
step.ExecutionContext.Section(StringUtil.Loc("StepFinishing", step.DisplayName));
step.ExecutionContext.Complete();
}
private async Task RunStepsAsync(List <IStep> embeddedSteps, ActionRunStage stage)
{
ArgUtil.NotNull(embeddedSteps, nameof(embeddedSteps));
foreach (IStep step in embeddedSteps)
{
Trace.Info($"Processing embedded step: DisplayName='{step.DisplayName}'");
// Add Expression Functions
step.ExecutionContext.ExpressionFunctions.Add(new FunctionInfo <HashFilesFunction>(PipelineTemplateConstants.HashFiles, 1, byte.MaxValue));
step.ExecutionContext.ExpressionFunctions.Add(new FunctionInfo <AlwaysFunction>(PipelineTemplateConstants.Always, 0, 0));
step.ExecutionContext.ExpressionFunctions.Add(new FunctionInfo <CancelledFunction>(PipelineTemplateConstants.Cancelled, 0, 0));
step.ExecutionContext.ExpressionFunctions.Add(new FunctionInfo <FailureFunction>(PipelineTemplateConstants.Failure, 0, 0));
step.ExecutionContext.ExpressionFunctions.Add(new FunctionInfo <SuccessFunction>(PipelineTemplateConstants.Success, 0, 0));
// Set action_status to the success of the current composite action
var actionResult = ExecutionContext.Result?.ToActionResult() ?? ActionResult.Success;
step.ExecutionContext.SetGitHubContext("action_status", actionResult.ToString());
// Initialize env context
Trace.Info("Initialize Env context for embedded step");
#if OS_WINDOWS
var envContext = new DictionaryContextData();
#else
var envContext = new CaseSensitiveDictionaryContextData();
#endif
step.ExecutionContext.ExpressionValues["env"] = envContext;
// Merge global env
foreach (var pair in ExecutionContext.Global.EnvironmentVariables)
{
envContext[pair.Key] = new StringContextData(pair.Value ?? string.Empty);
}
// Merge composite-step env
if (ExecutionContext.ExpressionValues.TryGetValue("env", out var envContextData))
{
#if OS_WINDOWS
var dict = envContextData as DictionaryContextData;
#else
var dict = envContextData as CaseSensitiveDictionaryContextData;
#endif
foreach (var pair in dict)
{
envContext[pair.Key] = pair.Value;
}
}
try
{
if (step is IActionRunner actionStep)
{
// Evaluate and merge embedded-step env
var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator();
var actionEnvironment = templateEvaluator.EvaluateStepEnvironment(actionStep.Action.Environment, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, Common.Util.VarUtil.EnvironmentVariableKeyComparer);
foreach (var env in actionEnvironment)
{
envContext[env.Key] = new StringContextData(env.Value ?? string.Empty);
}
}
}
catch (Exception ex)
{
// Evaluation error
Trace.Info("Caught exception from expression for embedded step.env");
step.ExecutionContext.Error(ex);
step.ExecutionContext.Complete(TaskResult.Failed);
}
// Register Callback
CancellationTokenRegistration?jobCancelRegister = null;
try
{
// Register job cancellation call back only if job cancellation token not been fire before each step run
if (!ExecutionContext.Root.CancellationToken.IsCancellationRequested)
{
// Test the condition again. The job was canceled after the condition was originally evaluated.
jobCancelRegister = ExecutionContext.Root.CancellationToken.Register(() =>
{
// Mark job as cancelled
ExecutionContext.Root.Result = TaskResult.Canceled;
ExecutionContext.Root.JobContext.Status = ExecutionContext.Root.Result?.ToActionResult();
step.ExecutionContext.Debug($"Re-evaluate condition on job cancellation for step: '{step.DisplayName}'.");
var conditionReTestTraceWriter = new ConditionTraceWriter(Trace, null); // host tracing only
var conditionReTestResult = false;
if (HostContext.RunnerShutdownToken.IsCancellationRequested)
{
step.ExecutionContext.Debug($"Skip Re-evaluate condition on runner shutdown.");
}
else
{
try
{
var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(conditionReTestTraceWriter);
var condition = new BasicExpressionToken(null, null, null, step.Condition);
conditionReTestResult = templateEvaluator.EvaluateStepIf(condition, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, step.ExecutionContext.ToExpressionState());
}
catch (Exception ex)
{
// Cancel the step since we get exception while re-evaluate step condition
Trace.Info("Caught exception from expression when re-test condition on job cancellation.");
step.ExecutionContext.Error(ex);
}
}
if (!conditionReTestResult)
{
// Cancel the step
Trace.Info("Cancel current running step.");
step.ExecutionContext.CancelToken();
}
});
}
else
{
if (ExecutionContext.Root.Result != TaskResult.Canceled)
{
// Mark job as cancelled
ExecutionContext.Root.Result = TaskResult.Canceled;
ExecutionContext.Root.JobContext.Status = ExecutionContext.Root.Result?.ToActionResult();
}
}
// Evaluate condition
step.ExecutionContext.Debug($"Evaluating condition for step: '{step.DisplayName}'");
var conditionTraceWriter = new ConditionTraceWriter(Trace, step.ExecutionContext);
var conditionResult = false;
var conditionEvaluateError = default(Exception);
if (HostContext.RunnerShutdownToken.IsCancellationRequested)
{
step.ExecutionContext.Debug($"Skip evaluate condition on runner shutdown.");
}
else
{
try
{
var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(conditionTraceWriter);
var condition = new BasicExpressionToken(null, null, null, step.Condition);
conditionResult = templateEvaluator.EvaluateStepIf(condition, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, step.ExecutionContext.ToExpressionState());
}
catch (Exception ex)
{
Trace.Info("Caught exception from expression.");
Trace.Error(ex);
conditionEvaluateError = ex;
}
}
if (!conditionResult && conditionEvaluateError == null)
{
// Condition is false
Trace.Info("Skipping step due to condition evaluation.");
step.ExecutionContext.Result = TaskResult.Skipped;
continue;
}
else if (conditionEvaluateError != null)
{
// Condition error
step.ExecutionContext.Error(conditionEvaluateError);
step.ExecutionContext.Result = TaskResult.Failed;
ExecutionContext.Result = TaskResult.Failed;
break;
}
else
{
await RunStepAsync(step);
}
}
finally
{
if (jobCancelRegister != null)
{
jobCancelRegister?.Dispose();
jobCancelRegister = null;
}
}
// Check failed or canceled
if (step.ExecutionContext.Result == TaskResult.Failed || step.ExecutionContext.Result == TaskResult.Canceled)
{
Trace.Info($"Update job result with current composite step result '{step.ExecutionContext.Result}'.");
ExecutionContext.Result = TaskResultUtil.MergeTaskResults(ExecutionContext.Result, step.ExecutionContext.Result.Value);
}
}
}
// StepsRunner should never throw exception to caller
public async Task RunAsync(IExecutionContext jobContext)
{
ArgUtil.NotNull(jobContext, nameof(jobContext));
ArgUtil.NotNull(jobContext.JobSteps, nameof(jobContext.JobSteps));
// TaskResult:
// Abandoned (Server set this.)
// Canceled
// Failed
// Skipped
// Succeeded
CancellationTokenRegistration?jobCancelRegister = null;
jobContext.JobContext.Status = (jobContext.Result ?? TaskResult.Succeeded).ToActionResult();
var scopeInputs = new Dictionary <string, PipelineContextData>(StringComparer.OrdinalIgnoreCase);
bool checkPostJobActions = false;
while (jobContext.JobSteps.Count > 0 || !checkPostJobActions)
{
if (jobContext.JobSteps.Count == 0 && !checkPostJobActions)
{
checkPostJobActions = true;
while (jobContext.PostJobSteps.TryPop(out var postStep))
{
jobContext.JobSteps.Enqueue(postStep);
}
continue;
}
var step = jobContext.JobSteps.Dequeue();
IStep nextStep = null;
if (jobContext.JobSteps.Count > 0)
{
nextStep = jobContext.JobSteps.Peek();
}
Trace.Info($"Processing step: DisplayName='{step.DisplayName}'");
ArgUtil.NotNull(step.ExecutionContext, nameof(step.ExecutionContext));
ArgUtil.NotNull(step.ExecutionContext.Variables, nameof(step.ExecutionContext.Variables));
// Start
step.ExecutionContext.Start();
// Initialize scope
if (InitializeScope(step, scopeInputs))
{
// Populate env context for each step
Trace.Info("Initialize Env context for step");
#if OS_WINDOWS
var envContext = new DictionaryContextData();
#else
var envContext = new CaseSensitiveDictionaryContextData();
#endif
step.ExecutionContext.ExpressionValues["env"] = envContext;
foreach (var pair in step.ExecutionContext.EnvironmentVariables)
{
envContext[pair.Key] = new StringContextData(pair.Value ?? string.Empty);
}
if (step is IActionRunner actionStep)
{
// Set GITHUB_ACTION
step.ExecutionContext.SetGitHubContext("action", actionStep.Action.Name);
// Evaluate and merge action's env block to env context
var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator();
var actionEnvironment = templateEvaluator.EvaluateStepEnvironment(actionStep.Action.Environment, step.ExecutionContext.ExpressionValues, VarUtil.EnvironmentVariableKeyComparer);
foreach (var env in actionEnvironment)
{
envContext[env.Key] = new StringContextData(env.Value ?? string.Empty);
}
}
var expressionManager = HostContext.GetService <IExpressionManager>();
try
{
// Register job cancellation call back only if job cancellation token not been fire before each step run
if (!jobContext.CancellationToken.IsCancellationRequested)
{
// Test the condition again. The job was canceled after the condition was originally evaluated.
jobCancelRegister = jobContext.CancellationToken.Register(() =>
{
// mark job as cancelled
jobContext.Result = TaskResult.Canceled;
jobContext.JobContext.Status = jobContext.Result?.ToActionResult();
step.ExecutionContext.Debug($"Re-evaluate condition on job cancellation for step: '{step.DisplayName}'.");
ConditionResult conditionReTestResult;
if (HostContext.RunnerShutdownToken.IsCancellationRequested)
{
step.ExecutionContext.Debug($"Skip Re-evaluate condition on runner shutdown.");
conditionReTestResult = false;
}
else
{
try
{
conditionReTestResult = expressionManager.Evaluate(step.ExecutionContext, step.Condition, hostTracingOnly: true);
}
catch (Exception ex)
{
// Cancel the step since we get exception while re-evaluate step condition.
Trace.Info("Caught exception from expression when re-test condition on job cancellation.");
step.ExecutionContext.Error(ex);
conditionReTestResult = false;
}
}
if (!conditionReTestResult.Value)
{
// Cancel the step.
Trace.Info("Cancel current running step.");
step.ExecutionContext.CancelToken();
}
});
}
else
{
if (jobContext.Result != TaskResult.Canceled)
{
// mark job as cancelled
jobContext.Result = TaskResult.Canceled;
jobContext.JobContext.Status = jobContext.Result?.ToActionResult();
}
}
// Evaluate condition.
step.ExecutionContext.Debug($"Evaluating condition for step: '{step.DisplayName}'");
Exception conditionEvaluateError = null;
ConditionResult conditionResult;
if (HostContext.RunnerShutdownToken.IsCancellationRequested)
{
step.ExecutionContext.Debug($"Skip evaluate condition on runner shutdown.");
conditionResult = false;
}
else
{
try
{
conditionResult = expressionManager.Evaluate(step.ExecutionContext, step.Condition);
}
catch (Exception ex)
{
Trace.Info("Caught exception from expression.");
Trace.Error(ex);
conditionResult = false;
conditionEvaluateError = ex;
}
}
// no evaluate error but condition is false
if (!conditionResult.Value && conditionEvaluateError == null)
{
// Condition == false
Trace.Info("Skipping step due to condition evaluation.");
CompleteStep(step, nextStep, TaskResult.Skipped, resultCode: conditionResult.Trace);
}
else if (conditionEvaluateError != null)
{
// fail the step since there is an evaluate error.
step.ExecutionContext.Error(conditionEvaluateError);
CompleteStep(step, nextStep, TaskResult.Failed);
}
else
{
// Run the step.
await RunStepAsync(step, jobContext.CancellationToken);
CompleteStep(step, nextStep);
}
}
finally
{
if (jobCancelRegister != null)
{
jobCancelRegister?.Dispose();
jobCancelRegister = null;
}
}
}
// Update the job result.
if (step.ExecutionContext.Result == TaskResult.Failed)
{
Trace.Info($"Update job result with current step result '{step.ExecutionContext.Result}'.");
jobContext.Result = TaskResultUtil.MergeTaskResults(jobContext.Result, step.ExecutionContext.Result.Value);
jobContext.JobContext.Status = jobContext.Result?.ToActionResult();
}
else
{
Trace.Info($"No need for updating job result with current step result '{step.ExecutionContext.Result}'.");
}
Trace.Info($"Current state: job state = '{jobContext.Result}'");
}
}
public async Task <int> RunAsync(string pipeIn, string pipeOut)
{
try
{
// Setup way to handle SIGTERM/unloading signals
_completedCommand.Reset();
HostContext.Unloading += Worker_Unloading;
// Validate args.
ArgUtil.NotNullOrEmpty(pipeIn, nameof(pipeIn));
ArgUtil.NotNullOrEmpty(pipeOut, nameof(pipeOut));
VssUtil.InitializeVssClientSettings(HostContext.UserAgents, HostContext.WebProxy);
var jobRunner = HostContext.CreateService <IJobRunner>();
using (var channel = HostContext.CreateService <IProcessChannel>())
using (var jobRequestCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(HostContext.RunnerShutdownToken))
using (var channelTokenSource = new CancellationTokenSource())
{
// Start the channel.
channel.StartClient(pipeIn, pipeOut);
// Wait for up to 30 seconds for a message from the channel.
HostContext.WritePerfCounter("WorkerWaitingForJobMessage");
Trace.Info("Waiting to receive the job message from the channel.");
WorkerMessage channelMessage;
using (var csChannelMessage = new CancellationTokenSource(_workerStartTimeout))
{
channelMessage = await channel.ReceiveAsync(csChannelMessage.Token);
}
// Deserialize the job message.
Trace.Info("Message received.");
ArgUtil.Equal(MessageType.NewJobRequest, channelMessage.MessageType, nameof(channelMessage.MessageType));
ArgUtil.NotNullOrEmpty(channelMessage.Body, nameof(channelMessage.Body));
var jobMessage = StringUtil.ConvertFromJson <Pipelines.AgentJobRequestMessage>(channelMessage.Body);
ArgUtil.NotNull(jobMessage, nameof(jobMessage));
HostContext.WritePerfCounter($"WorkerJobMessageReceived_{jobMessage.RequestId.ToString()}");
// Initialize the secret masker and set the thread culture.
InitializeSecretMasker(jobMessage);
SetCulture(jobMessage);
// Start the job.
Trace.Info($"Job message:{Environment.NewLine} {StringUtil.ConvertToJson(jobMessage)}");
Task <TaskResult> jobRunnerTask = jobRunner.RunAsync(jobMessage, jobRequestCancellationToken.Token);
// Start listening for a cancel message from the channel.
Trace.Info("Listening for cancel message from the channel.");
Task <WorkerMessage> channelTask = channel.ReceiveAsync(channelTokenSource.Token);
// Wait for one of the tasks to complete.
Trace.Info("Waiting for the job to complete or for a cancel message from the channel.");
Task.WaitAny(jobRunnerTask, channelTask);
// Handle if the job completed.
if (jobRunnerTask.IsCompleted)
{
Trace.Info("Job completed.");
channelTokenSource.Cancel(); // Cancel waiting for a message from the channel.
return(TaskResultUtil.TranslateToReturnCode(await jobRunnerTask));
}
// Otherwise a cancel message was received from the channel.
Trace.Info("Cancellation/Shutdown message received.");
channelMessage = await channelTask;
switch (channelMessage.MessageType)
{
case MessageType.CancelRequest:
jobRequestCancellationToken.Cancel(); // Expire the host cancellation token.
break;
case MessageType.RunnerShutdown:
HostContext.ShutdownRunner(ShutdownReason.UserCancelled);
break;
case MessageType.OperatingSystemShutdown:
HostContext.ShutdownRunner(ShutdownReason.OperatingSystemShutdown);
break;
default:
throw new ArgumentOutOfRangeException(nameof(channelMessage.MessageType), channelMessage.MessageType, nameof(channelMessage.MessageType));
}
// Await the job.
return(TaskResultUtil.TranslateToReturnCode(await jobRunnerTask));
}
}
finally
{
HostContext.Unloading -= Worker_Unloading;
_completedCommand.Set();
}
}
private async Task RunStepAsync(IStep step, CancellationToken jobCancellationToken)
{
// Start the step.
Trace.Info("Starting the step.");
step.ExecutionContext.Section(StringUtil.Loc("StepStarting", step.DisplayName));
step.ExecutionContext.SetTimeout(timeout: step.Timeout);
step.ExecutionContext.SetStepTarget(step.Target);
// Windows may not be on the UTF8 codepage; try to fix that
await SwitchToUtf8Codepage(step);
try
{
await step.RunAsync();
}
catch (OperationCanceledException ex)
{
if (step.ExecutionContext.CancellationToken.IsCancellationRequested &&
!jobCancellationToken.IsCancellationRequested)
{
Trace.Error($"Caught timeout exception from step: {ex.Message}");
step.ExecutionContext.Error(StringUtil.Loc("StepTimedOut"));
step.ExecutionContext.Result = TaskResult.Failed;
}
else
{
// Log the exception and cancel the step.
Trace.Error($"Caught cancellation exception from step: {ex}");
step.ExecutionContext.Error(ex);
step.ExecutionContext.Result = TaskResult.Canceled;
}
}
catch (Exception ex)
{
// Log the error and fail the step.
Trace.Error($"Caught exception from step: {ex}");
step.ExecutionContext.Error(ex);
step.ExecutionContext.Result = TaskResult.Failed;
}
// Wait till all async commands finish.
foreach (var command in step.ExecutionContext.AsyncCommands ?? new List <IAsyncCommandContext>())
{
try
{
// wait async command to finish.
await command.WaitAsync();
}
catch (OperationCanceledException ex)
{
if (step.ExecutionContext.CancellationToken.IsCancellationRequested &&
!jobCancellationToken.IsCancellationRequested)
{
// Log the timeout error, set step result to falied if the current result is not canceled.
Trace.Error($"Caught timeout exception from async command {command.Name}: {ex}");
step.ExecutionContext.Error(StringUtil.Loc("StepTimedOut"));
// if the step already canceled, don't set it to failed.
step.ExecutionContext.CommandResult = TaskResultUtil.MergeTaskResults(step.ExecutionContext.CommandResult, TaskResult.Failed);
}
else
{
// log and save the OperationCanceledException, set step result to canceled if the current result is not failed.
Trace.Error($"Caught cancellation exception from async command {command.Name}: {ex}");
step.ExecutionContext.Error(ex);
// if the step already failed, don't set it to canceled.
step.ExecutionContext.CommandResult = TaskResultUtil.MergeTaskResults(step.ExecutionContext.CommandResult, TaskResult.Canceled);
}
}
catch (Exception ex)
{
// Log the error, set step result to falied if the current result is not canceled.
Trace.Error($"Caught exception from async command {command.Name}: {ex}");
step.ExecutionContext.Error(ex);
// if the step already canceled, don't set it to failed.
step.ExecutionContext.CommandResult = TaskResultUtil.MergeTaskResults(step.ExecutionContext.CommandResult, TaskResult.Failed);
}
}
// Merge executioncontext result with command result
if (step.ExecutionContext.CommandResult != null)
{
step.ExecutionContext.Result = TaskResultUtil.MergeTaskResults(step.ExecutionContext.Result, step.ExecutionContext.CommandResult.Value);
}
// Fixup the step result if ContinueOnError.
if (step.ExecutionContext.Result == TaskResult.Failed && step.ContinueOnError)
{
step.ExecutionContext.Result = TaskResult.SucceededWithIssues;
Trace.Info($"Updated step result: {step.ExecutionContext.Result}");
}
else
{
Trace.Info($"Step result: {step.ExecutionContext.Result}");
}
// Complete the step context.
step.ExecutionContext.Section(StringUtil.Loc("StepFinishing", step.DisplayName));
step.ExecutionContext.Complete();
}
private async Task RunStepAsync(IStep step, CancellationToken jobCancellationToken)
{
// Check to see if we can expand the display name
if (step is IActionRunner actionRunner &&
actionRunner.Stage == ActionRunStage.Main &&
actionRunner.TryEvaluateDisplayName(step.ExecutionContext.ExpressionValues, step.ExecutionContext))
{
step.ExecutionContext.UpdateTimelineRecordDisplayName(actionRunner.DisplayName);
}
// Start the step.
Trace.Info("Starting the step.");
step.ExecutionContext.Debug($"Starting: {step.DisplayName}");
// Set the timeout
var timeoutMinutes = 0;
var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator();
try
{
timeoutMinutes = templateEvaluator.EvaluateStepTimeout(step.Timeout, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions);
}
catch (Exception ex)
{
Trace.Info("An error occurred when attempting to determine the step timeout.");
Trace.Error(ex);
step.ExecutionContext.Error("An error occurred when attempting to determine the step timeout.");
step.ExecutionContext.Error(ex);
}
if (timeoutMinutes > 0)
{
var timeout = TimeSpan.FromMinutes(timeoutMinutes);
step.ExecutionContext.SetTimeout(timeout);
}
await EncodingUtil.SetEncoding(HostContext, Trace, step.ExecutionContext.CancellationToken);
try
{
await step.RunAsync();
}
catch (OperationCanceledException ex)
{
if (step.ExecutionContext.CancellationToken.IsCancellationRequested &&
!jobCancellationToken.IsCancellationRequested)
{
Trace.Error($"Caught timeout exception from step: {ex.Message}");
step.ExecutionContext.Error("The action has timed out.");
step.ExecutionContext.Result = TaskResult.Failed;
}
else
{
// Log the exception and cancel the step.
Trace.Error($"Caught cancellation exception from step: {ex}");
step.ExecutionContext.Error(ex);
step.ExecutionContext.Result = TaskResult.Canceled;
}
}
catch (Exception ex)
{
// Log the error and fail the step.
Trace.Error($"Caught exception from step: {ex}");
step.ExecutionContext.Error(ex);
step.ExecutionContext.Result = TaskResult.Failed;
}
// Merge execution context result with command result
if (step.ExecutionContext.CommandResult != null)
{
step.ExecutionContext.Result = TaskResultUtil.MergeTaskResults(step.ExecutionContext.Result, step.ExecutionContext.CommandResult.Value);
}
// Fixup the step result if ContinueOnError.
if (step.ExecutionContext.Result == TaskResult.Failed)
{
var continueOnError = false;
try
{
continueOnError = templateEvaluator.EvaluateStepContinueOnError(step.ContinueOnError, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions);
}
catch (Exception ex)
{
Trace.Info("The step failed and an error occurred when attempting to determine whether to continue on error.");
Trace.Error(ex);
step.ExecutionContext.Error("The step failed and an error occurred when attempting to determine whether to continue on error.");
step.ExecutionContext.Error(ex);
}
if (continueOnError)
{
step.ExecutionContext.Outcome = step.ExecutionContext.Result;
step.ExecutionContext.Result = TaskResult.Succeeded;
Trace.Info($"Updated step result (continue on error)");
}
}
Trace.Info($"Step result: {step.ExecutionContext.Result}");
// Complete the step context.
step.ExecutionContext.Debug($"Finishing: {step.DisplayName}");
}
public void FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, DateTime jobStartTimeUtc)
{
Trace.Entering();
ArgUtil.NotNull(jobContext, nameof(jobContext));
// create a new timeline record node for 'Finalize job'
IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), "Complete job", $"{nameof(JobExtension)}_Final", null, null);
using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
{
try
{
context.Start();
context.Debug("Starting: Complete job");
Trace.Info("Initialize Env context");
#if OS_WINDOWS
var envContext = new DictionaryContextData();
#else
var envContext = new CaseSensitiveDictionaryContextData();
#endif
context.ExpressionValues["env"] = envContext;
foreach (var pair in context.Global.EnvironmentVariables)
{
envContext[pair.Key] = new StringContextData(pair.Value ?? string.Empty);
}
// Populate env context for each step
Trace.Info("Initialize steps context");
context.ExpressionValues["steps"] = context.Global.StepsContext.GetScope(context.ScopeName);
var templateEvaluator = context.ToPipelineTemplateEvaluator();
// Evaluate job outputs
if (message.JobOutputs != null && message.JobOutputs.Type != TokenType.Null)
{
try
{
context.Output($"Evaluate and set job outputs");
// Populate env context for each step
Trace.Info("Initialize Env context for evaluating job outputs");
var outputs = templateEvaluator.EvaluateJobOutput(message.JobOutputs, context.ExpressionValues, context.ExpressionFunctions);
foreach (var output in outputs)
{
if (string.IsNullOrEmpty(output.Value))
{
context.Debug($"Skip output '{output.Key}' since it's empty");
continue;
}
if (!string.Equals(output.Value, HostContext.SecretMasker.MaskSecrets(output.Value)))
{
context.Warning($"Skip output '{output.Key}' since it may contain secret.");
continue;
}
context.Output($"Set output '{output.Key}'");
jobContext.JobOutputs[output.Key] = output.Value;
}
}
catch (Exception ex)
{
context.Result = TaskResult.Failed;
context.Error($"Fail to evaluate job outputs");
context.Error(ex);
jobContext.Result = TaskResultUtil.MergeTaskResults(jobContext.Result, TaskResult.Failed);
}
}
// Evaluate environment data
if (jobContext.ActionsEnvironment?.Url != null && jobContext.ActionsEnvironment?.Url.Type != TokenType.Null)
{
try
{
context.Output($"Evaluate and set environment url");
var environmentUrlToken = templateEvaluator.EvaluateEnvironmentUrl(jobContext.ActionsEnvironment.Url, context.ExpressionValues, context.ExpressionFunctions);
var environmentUrl = environmentUrlToken.AssertString("environment.url");
if (!string.Equals(environmentUrl.Value, HostContext.SecretMasker.MaskSecrets(environmentUrl.Value)))
{
context.Warning($"Skip setting environment url as environment '{jobContext.ActionsEnvironment.Name}' may contain secret.");
}
else
{
context.Output($"Evaluated environment url: {environmentUrl}");
jobContext.ActionsEnvironment.Url = environmentUrlToken;
}
}
catch (Exception ex)
{
context.Result = TaskResult.Failed;
context.Error($"Failed to evaluate environment url");
context.Error(ex);
jobContext.Result = TaskResultUtil.MergeTaskResults(jobContext.Result, TaskResult.Failed);
}
}
if (context.Global.Variables.GetBoolean(Constants.Variables.Actions.RunnerDebug) ?? false)
{
Trace.Info("Support log upload starting.");
context.Output("Uploading runner diagnostic logs");
IDiagnosticLogManager diagnosticLogManager = HostContext.GetService <IDiagnosticLogManager>();
try
{
diagnosticLogManager.UploadDiagnosticLogs(executionContext: context, parentContext: jobContext, message: message, jobStartTimeUtc: jobStartTimeUtc);
Trace.Info("Support log upload complete.");
context.Output("Completed runner diagnostic log upload");
}
catch (Exception ex)
{
// Log the error but make sure we continue gracefully.
Trace.Info("Error uploading support logs.");
context.Output("Error uploading runner diagnostic logs");
Trace.Error(ex);
}
}
if (_processCleanup)
{
context.Output("Cleaning up orphan processes");
// Only check environment variable for any process that doesn't run before we invoke our process.
Dictionary <int, Process> currentProcesses = SnapshotProcesses();
foreach (var proc in currentProcesses)
{
if (proc.Key == Process.GetCurrentProcess().Id)
{
// skip for current process.
continue;
}
if (_existingProcesses.Contains($"{proc.Key}_{proc.Value.ProcessName}"))
{
Trace.Verbose($"Skip existing process. PID: {proc.Key} ({proc.Value.ProcessName})");
}
else
{
Trace.Info($"Inspecting process environment variables. PID: {proc.Key} ({proc.Value.ProcessName})");
string lookupId = null;
try
{
lookupId = proc.Value.GetEnvironmentVariable(HostContext, Constants.ProcessTrackingId);
}
catch (Exception ex)
{
Trace.Warning($"Ignore exception during read process environment variables: {ex.Message}");
Trace.Verbose(ex.ToString());
}
if (string.Equals(lookupId, _processLookupId, StringComparison.OrdinalIgnoreCase))
{
context.Output($"Terminate orphan process: pid ({proc.Key}) ({proc.Value.ProcessName})");
try
{
proc.Value.Kill();
}
catch (Exception ex)
{
Trace.Error("Catch exception during orphan process cleanup.");
Trace.Error(ex);
}
}
}
}
}
if (_diskSpaceCheckTask != null)
{
_diskSpaceCheckToken.Cancel();
}
}
catch (Exception ex)
{
// Log and ignore the error from JobExtension finalization.
Trace.Error($"Caught exception from JobExtension finalization: {ex}");
context.Output(ex.Message);
}
finally
{
context.Debug("Finishing: Complete job");
context.Complete();
}
}
}
public async Task <int> RunAsync(string pipeIn, string pipeOut)
{
// Validate args.
ArgUtil.NotNullOrEmpty(pipeIn, nameof(pipeIn));
ArgUtil.NotNullOrEmpty(pipeOut, nameof(pipeOut));
var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>();
var agentCertManager = HostContext.GetService <IAgentCertificateManager>();
VssUtil.InitializeVssClientSettings(HostContext.UserAgent, agentWebProxy.WebProxy, agentCertManager.VssClientCertificateManager);
var jobRunner = HostContext.CreateService <IJobRunner>();
using (var channel = HostContext.CreateService <IProcessChannel>())
using (var jobRequestCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(HostContext.AgentShutdownToken))
using (var channelTokenSource = new CancellationTokenSource())
{
// Start the channel.
channel.StartClient(pipeIn, pipeOut);
// Wait for up to 30 seconds for a message from the channel.
HostContext.WritePerfCounter("WorkerWaitingForJobMessage");
Trace.Info("Waiting to receive the job message from the channel.");
WorkerMessage channelMessage;
using (var csChannelMessage = new CancellationTokenSource(_workerStartTimeout))
{
channelMessage = await channel.ReceiveAsync(csChannelMessage.Token);
}
// Deserialize the job message.
Trace.Info("Message received.");
ArgUtil.Equal(MessageType.NewJobRequest, channelMessage.MessageType, nameof(channelMessage.MessageType));
ArgUtil.NotNullOrEmpty(channelMessage.Body, nameof(channelMessage.Body));
var jobMessage = JsonUtility.FromString <Pipelines.AgentJobRequestMessage>(channelMessage.Body);
ArgUtil.NotNull(jobMessage, nameof(jobMessage));
HostContext.WritePerfCounter($"WorkerJobMessageReceived_{jobMessage.RequestId.ToString()}");
// Initialize the secret masker and set the thread culture.
InitializeSecretMasker(jobMessage);
SetCulture(jobMessage);
// Start the job.
Trace.Info($"Job message:{Environment.NewLine} {StringUtil.ConvertToJson(WorkerUtilities.ScrubPiiData(jobMessage))}");
Task <TaskResult> jobRunnerTask = jobRunner.RunAsync(jobMessage, jobRequestCancellationToken.Token);
bool cancel = false;
while (!cancel)
{
// Start listening for a cancel message from the channel.
Trace.Info("Listening for cancel message from the channel.");
Task <WorkerMessage> channelTask = channel.ReceiveAsync(channelTokenSource.Token);
// Wait for one of the tasks to complete.
Trace.Info("Waiting for the job to complete or for a cancel message from the channel.");
await Task.WhenAny(jobRunnerTask, channelTask);
// Handle if the job completed.
if (jobRunnerTask.IsCompleted)
{
Trace.Info("Job completed.");
channelTokenSource.Cancel(); // Cancel waiting for a message from the channel.
return(TaskResultUtil.TranslateToReturnCode(await jobRunnerTask));
}
// Otherwise a message was received from the channel.
channelMessage = await channelTask;
switch (channelMessage.MessageType)
{
case MessageType.CancelRequest:
Trace.Info("Cancellation/Shutdown message received.");
cancel = true;
jobRequestCancellationToken.Cancel(); // Expire the host cancellation token.
break;
case MessageType.AgentShutdown:
Trace.Info("Cancellation/Shutdown message received.");
cancel = true;
HostContext.ShutdownAgent(ShutdownReason.UserCancelled);
break;
case MessageType.OperatingSystemShutdown:
Trace.Info("Cancellation/Shutdown message received.");
cancel = true;
HostContext.ShutdownAgent(ShutdownReason.OperatingSystemShutdown);
break;
case MessageType.JobMetadataUpdate:
Trace.Info("Metadata update message received.");
var metadataMessage = JsonUtility.FromString <JobMetadataMessage>(channelMessage.Body);
jobRunner.UpdateMetadata(metadataMessage);
break;
default:
throw new ArgumentOutOfRangeException(nameof(channelMessage.MessageType), channelMessage.MessageType, nameof(channelMessage.MessageType));
}
}
// Await the job.
return(TaskResultUtil.TranslateToReturnCode(await jobRunnerTask));
}
}
public async Task <int> RunAsync(string pipeIn, string pipeOut)
{
// Validate args.
ArgUtil.NotNullOrEmpty(pipeIn, nameof(pipeIn));
ArgUtil.NotNullOrEmpty(pipeOut, nameof(pipeOut));
var proxyConfig = HostContext.GetService <IProxyConfiguration>();
proxyConfig.ApplyProxySettings();
var jobRunner = HostContext.CreateService <IJobRunner>();
using (var channel = HostContext.CreateService <IProcessChannel>())
using (var jobRequestCancellationToken = new CancellationTokenSource())
using (var channelTokenSource = new CancellationTokenSource())
{
// Start the channel.
channel.StartClient(pipeIn, pipeOut);
// Wait for up to 30 seconds for a message from the channel.
Trace.Info("Waiting to receive the job message from the channel.");
WorkerMessage channelMessage;
using (var csChannelMessage = new CancellationTokenSource(_workerStartTimeout))
{
channelMessage = await channel.ReceiveAsync(csChannelMessage.Token);
}
// Deserialize the job message.
Trace.Info("Message received.");
ArgUtil.Equal(MessageType.NewJobRequest, channelMessage.MessageType, nameof(channelMessage.MessageType));
ArgUtil.NotNullOrEmpty(channelMessage.Body, nameof(channelMessage.Body));
var jobMessage = JsonUtility.FromString <AgentJobRequestMessage>(channelMessage.Body);
ArgUtil.NotNull(jobMessage, nameof(jobMessage));
// Initialize the secret masker and set the thread culture.
InitializeSecretMasker(jobMessage);
SetCulture(jobMessage);
// Start the job.
Trace.Info($"Job message:{Environment.NewLine} {StringUtil.ConvertToJson(jobMessage)}");
Task <TaskResult> jobRunnerTask = jobRunner.RunAsync(jobMessage, jobRequestCancellationToken.Token);
// Start listening for a cancel message from the channel.
Trace.Info("Listening for cancel message from the channel.");
Task <WorkerMessage> channelTask = channel.ReceiveAsync(channelTokenSource.Token);
// Wait for one of the tasks to complete.
Trace.Info("Waiting for the job to complete or for a cancel message from the channel.");
Task.WaitAny(jobRunnerTask, channelTask);
// Handle if the job completed.
if (jobRunnerTask.IsCompleted)
{
Trace.Info("Job completed.");
channelTokenSource.Cancel(); // Cancel waiting for a message from the channel.
return(TaskResultUtil.TranslateToReturnCode(await jobRunnerTask));
}
// Otherwise a cancel message was received from the channel.
Trace.Info("Cancellation message received.");
channelMessage = await channelTask;
ArgUtil.Equal(MessageType.CancelRequest, channelMessage.MessageType, nameof(channelMessage.MessageType));
jobRequestCancellationToken.Cancel(); // Expire the host cancellation token.
// Await the job.
return(TaskResultUtil.TranslateToReturnCode(await jobRunnerTask));
}
}
private async Task RunStepAsync(IStep step, CancellationToken jobCancellationToken)
{
// Check to see if we can expand the display name
if (step is IActionRunner actionRunner &&
actionRunner.Stage == ActionRunStage.Main &&
actionRunner.TryEvaluateDisplayName(step.ExecutionContext.ExpressionValues, step.ExecutionContext))
{
step.ExecutionContext.UpdateTimelineRecordDisplayName(actionRunner.DisplayName);
}
// Start the step.
Trace.Info("Starting the step.");
step.ExecutionContext.Debug($"Starting: {step.DisplayName}");
// Set the timeout
var timeoutMinutes = 0;
var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator();
try
{
timeoutMinutes = templateEvaluator.EvaluateStepTimeout(step.Timeout, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions);
}
catch (Exception ex)
{
Trace.Info("An error occurred when attempting to determine the step timeout.");
Trace.Error(ex);
step.ExecutionContext.Error("An error occurred when attempting to determine the step timeout.");
step.ExecutionContext.Error(ex);
}
if (timeoutMinutes > 0)
{
var timeout = TimeSpan.FromMinutes(timeoutMinutes);
step.ExecutionContext.SetTimeout(timeout);
}
#if OS_WINDOWS
try
{
if (Console.InputEncoding.CodePage != 65001)
{
using (var p = HostContext.CreateService <IProcessInvoker>())
{
// Use UTF8 code page
int exitCode = await p.ExecuteAsync(workingDirectory : HostContext.GetDirectory(WellKnownDirectory.Work),
fileName : WhichUtil.Which("chcp", true, Trace),
arguments : "65001",
environment : null,
requireExitCodeZero : false,
outputEncoding : null,
killProcessOnCancel : false,
redirectStandardIn : null,
inheritConsoleHandler : true,
cancellationToken : step.ExecutionContext.CancellationToken);
if (exitCode == 0)
{
Trace.Info("Successfully returned to code page 65001 (UTF8)");
}
else
{
Trace.Warning($"'chcp 65001' failed with exit code {exitCode}");
}
}
}
}
catch (Exception ex)
{
Trace.Warning($"'chcp 65001' failed with exception {ex.Message}");
}
#endif
try
{
await step.RunAsync();
}
catch (OperationCanceledException ex)
{
if (step.ExecutionContext.CancellationToken.IsCancellationRequested &&
!jobCancellationToken.IsCancellationRequested)
{
Trace.Error($"Caught timeout exception from step: {ex.Message}");
step.ExecutionContext.Error("The action has timed out.");
step.ExecutionContext.Result = TaskResult.Failed;
}
else
{
// Log the exception and cancel the step.
Trace.Error($"Caught cancellation exception from step: {ex}");
step.ExecutionContext.Error(ex);
step.ExecutionContext.Result = TaskResult.Canceled;
}
}
catch (Exception ex)
{
// Log the error and fail the step.
Trace.Error($"Caught exception from step: {ex}");
step.ExecutionContext.Error(ex);
step.ExecutionContext.Result = TaskResult.Failed;
}
// Merge execution context result with command result
if (step.ExecutionContext.CommandResult != null)
{
step.ExecutionContext.Result = TaskResultUtil.MergeTaskResults(step.ExecutionContext.Result, step.ExecutionContext.CommandResult.Value);
}
// Fixup the step result if ContinueOnError.
if (step.ExecutionContext.Result == TaskResult.Failed)
{
var continueOnError = false;
try
{
continueOnError = templateEvaluator.EvaluateStepContinueOnError(step.ContinueOnError, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions);
}
catch (Exception ex)
{
Trace.Info("The step failed and an error occurred when attempting to determine whether to continue on error.");
Trace.Error(ex);
step.ExecutionContext.Error("The step failed and an error occurred when attempting to determine whether to continue on error.");
step.ExecutionContext.Error(ex);
}
if (continueOnError)
{
step.ExecutionContext.Outcome = step.ExecutionContext.Result;
step.ExecutionContext.Result = TaskResult.Succeeded;
Trace.Info($"Updated step result (continue on error)");
}
}
Trace.Info($"Step result: {step.ExecutionContext.Result}");
// Complete the step context.
step.ExecutionContext.Debug($"Finishing: {step.DisplayName}");
}
private async Task RunAsync(JobRequestMessage message, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken)
{
if (previousJobDispatch != null)
{
Trace.Verbose($"Make sure the previous job request {previousJobDispatch.JobId} has successfully finished on worker.");
await EnsureDispatchFinished(previousJobDispatch);
}
else
{
Trace.Verbose($"This is the first job request.");
}
var term = HostContext.GetService <ITerminal>();
term.WriteLine(StringUtil.Loc("RunningJob", DateTime.UtcNow, message.JobName));
// first job request renew succeed.
TaskCompletionSource <int> firstJobRequestRenewed = new TaskCompletionSource <int>();
// lock renew cancellation token.
using (var lockRenewalTokenSource = new CancellationTokenSource())
using (var workerProcessCancelTokenSource = new CancellationTokenSource())
{
long requestId = message.RequestId;
Guid lockToken = message.LockToken;
// start renew job request
Trace.Info("Start renew job request.");
Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, firstJobRequestRenewed, lockRenewalTokenSource.Token);
// wait till first renew succeed or job request is canceled
// not even start worker if the first renew fail
await Task.WhenAny(firstJobRequestRenewed.Task, renewJobRequest, Task.Delay(-1, jobRequestCancellationToken));
if (renewJobRequest.IsCompleted)
{
// renew job request task complete means we run out of retry for the first job request renew.
// TODO: not need to return anything.
Trace.Info("Unable to renew job request for the first time, stop dispatching job to worker.");
return;
}
if (jobRequestCancellationToken.IsCancellationRequested)
{
await CompleteJobRequestAsync(_poolId, requestId, lockToken, TaskResult.Canceled);
return;
}
Task <int> workerProcessTask = null;
using (var processChannel = HostContext.CreateService <IProcessChannel>())
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
// Start the process channel.
// It's OK if StartServer bubbles an execption after the worker process has already started.
// The worker will shutdown after 30 seconds if it hasn't received the job message.
processChannel.StartServer(
// Delegate to start the child process.
startProcess: (string pipeHandleOut, string pipeHandleIn) =>
{
// Validate args.
ArgUtil.NotNullOrEmpty(pipeHandleOut, nameof(pipeHandleOut));
ArgUtil.NotNullOrEmpty(pipeHandleIn, nameof(pipeHandleIn));
// Start the child process.
var assemblyDirectory = IOUtil.GetBinPath();
string workerFileName = Path.Combine(assemblyDirectory, _workerProcessName);
workerProcessTask = processInvoker.ExecuteAsync(
workingDirectory: assemblyDirectory,
fileName: workerFileName,
arguments: "spawnclient " + pipeHandleOut + " " + pipeHandleIn,
environment: null,
cancellationToken: workerProcessCancelTokenSource.Token);
});
// Send the job request message.
// Kill the worker process if sending the job message times out. The worker
// process may have successfully received the job message.
try
{
Trace.Info("Send job request message to worker.");
using (var csSendJobRequest = new CancellationTokenSource(ChannelTimeout))
{
await processChannel.SendAsync(
messageType : MessageType.NewJobRequest,
body : JsonUtility.ToString(message),
cancellationToken : csSendJobRequest.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
// timeout 45 sec. kill worker.
Trace.Info("Job request message sending been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
// worker process been killed.
}
Trace.Info("Stop renew job request.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// not finish the job request since the job haven't run on worker at all, we will not going to set a result to server.
return;
}
TaskResult resultOnAbandonOrCancel = TaskResult.Succeeded;
// wait for renewlock, worker process or cancellation token been fired.
var completedTask = await Task.WhenAny(renewJobRequest, workerProcessTask, Task.Delay(-1, jobRequestCancellationToken));
if (completedTask == workerProcessTask)
{
// worker finished successfully, complete job request with result, stop renew lock, job has finished.
int returnCode = await workerProcessTask;
Trace.Info("Worker finished. Code: " + returnCode);
TaskResult result = TaskResultUtil.TranslateFromReturnCode(returnCode);
Trace.Info($"finish job request with result: {result}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobName, result));
// complete job request
await CompleteJobRequestAsync(_poolId, requestId, lockToken, result);
Trace.Info("Stop renew job request.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
return;
}
else if (completedTask == renewJobRequest)
{
resultOnAbandonOrCancel = TaskResult.Abandoned;
}
else
{
resultOnAbandonOrCancel = TaskResult.Canceled;
}
// renew job request completed or job request cancellation token been fired for RunAsync(jobrequestmessage)
// cancel worker gracefully first, then kill it after 45 sec
try
{
Trace.Info("Send job cancellation message to worker.");
using (var csSendCancel = new CancellationTokenSource(ChannelTimeout))
{
await processChannel.SendAsync(
messageType : MessageType.CancelRequest,
body : string.Empty,
cancellationToken : csSendCancel.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
Trace.Info("Job cancel message sending been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
// worker process been killed.
}
}
// wait worker to exit within 45 sec, then kill worker.
using (var csKillWorker = new CancellationTokenSource(TimeSpan.FromSeconds(45)))
{
completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, csKillWorker.Token));
}
// worker haven't exit within 45 sec.
if (completedTask != workerProcessTask)
{
Trace.Info("worker process haven't exit after 45 sec, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
// worker process been killed.
}
}
Trace.Info($"finish job request with result: {resultOnAbandonOrCancel}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobName, resultOnAbandonOrCancel));
// complete job request with cancel result, stop renew lock, job has finished.
//TODO: don't finish job request on abandon
await CompleteJobRequestAsync(_poolId, requestId, lockToken, resultOnAbandonOrCancel);
Trace.Info("Stop renew job request.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
}
}
}
// StepsRunner should never throw exception to caller
public async Task RunAsync(IExecutionContext jobContext)
{
ArgUtil.NotNull(jobContext, nameof(jobContext));
ArgUtil.NotNull(jobContext.JobSteps, nameof(jobContext.JobSteps));
// TaskResult:
// Abandoned (Server set this.)
// Canceled
// Failed
// Skipped
// Succeeded
CancellationTokenRegistration?jobCancelRegister = null;
jobContext.JobContext.Status = (jobContext.Result ?? TaskResult.Succeeded).ToActionResult();
var scopeInputs = new Dictionary <string, PipelineContextData>(StringComparer.OrdinalIgnoreCase);
bool checkPostJobActions = false;
while (jobContext.JobSteps.Count > 0 || !checkPostJobActions)
{
if (jobContext.JobSteps.Count == 0 && !checkPostJobActions)
{
checkPostJobActions = true;
while (jobContext.PostJobSteps.TryPop(out var postStep))
{
jobContext.JobSteps.Add(postStep);
}
continue;
}
var step = jobContext.JobSteps[0];
jobContext.JobSteps.RemoveAt(0);
Trace.Info($"Processing step: DisplayName='{step.DisplayName}'");
ArgUtil.NotNull(step.ExecutionContext, nameof(step.ExecutionContext));
ArgUtil.NotNull(step.ExecutionContext.Variables, nameof(step.ExecutionContext.Variables));
// Start
step.ExecutionContext.Start();
// Expression functions
step.ExecutionContext.ExpressionFunctions.Add(new FunctionInfo <AlwaysFunction>(PipelineTemplateConstants.Always, 0, 0));
step.ExecutionContext.ExpressionFunctions.Add(new FunctionInfo <CancelledFunction>(PipelineTemplateConstants.Cancelled, 0, 0));
step.ExecutionContext.ExpressionFunctions.Add(new FunctionInfo <FailureFunction>(PipelineTemplateConstants.Failure, 0, 0));
step.ExecutionContext.ExpressionFunctions.Add(new FunctionInfo <SuccessFunction>(PipelineTemplateConstants.Success, 0, 0));
step.ExecutionContext.ExpressionFunctions.Add(new FunctionInfo <HashFilesFunction>(PipelineTemplateConstants.HashFiles, 1, byte.MaxValue));
step.ExecutionContext.ExpressionValues["steps"] = step.ExecutionContext.StepsContext.GetScope(step.ExecutionContext.ScopeName);
// Populate env context for each step
Trace.Info("Initialize Env context for step");
#if OS_WINDOWS
var envContext = new DictionaryContextData();
#else
var envContext = new CaseSensitiveDictionaryContextData();
#endif
// Global env
foreach (var pair in step.ExecutionContext.EnvironmentVariables)
{
envContext[pair.Key] = new StringContextData(pair.Value ?? string.Empty);
}
// Stomps over with outside step env
if (step.ExecutionContext.ExpressionValues.TryGetValue("env", out var envContextData))
{
#if OS_WINDOWS
var dict = envContextData as DictionaryContextData;
#else
var dict = envContextData as CaseSensitiveDictionaryContextData;
#endif
foreach (var pair in dict)
{
envContext[pair.Key] = pair.Value;
}
}
step.ExecutionContext.ExpressionValues["env"] = envContext;
bool evaluateStepEnvFailed = false;
if (step is IActionRunner actionStep)
{
// Set GITHUB_ACTION
step.ExecutionContext.SetGitHubContext("action", actionStep.Action.Name);
try
{
// Evaluate and merge action's env block to env context
var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator();
var actionEnvironment = templateEvaluator.EvaluateStepEnvironment(actionStep.Action.Environment, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, VarUtil.EnvironmentVariableKeyComparer);
foreach (var env in actionEnvironment)
{
envContext[env.Key] = new StringContextData(env.Value ?? string.Empty);
}
}
catch (Exception ex)
{
// fail the step since there is an evaluate error.
Trace.Info("Caught exception from expression for step.env");
evaluateStepEnvFailed = true;
step.ExecutionContext.Error(ex);
CompleteStep(step, TaskResult.Failed);
}
}
if (!evaluateStepEnvFailed)
{
try
{
// Register job cancellation call back only if job cancellation token not been fire before each step run
if (!jobContext.CancellationToken.IsCancellationRequested)
{
// Test the condition again. The job was canceled after the condition was originally evaluated.
jobCancelRegister = jobContext.CancellationToken.Register(() =>
{
// mark job as cancelled
jobContext.Result = TaskResult.Canceled;
jobContext.JobContext.Status = jobContext.Result?.ToActionResult();
step.ExecutionContext.Debug($"Re-evaluate condition on job cancellation for step: '{step.DisplayName}'.");
var conditionReTestTraceWriter = new ConditionTraceWriter(Trace, null); // host tracing only
var conditionReTestResult = false;
if (HostContext.RunnerShutdownToken.IsCancellationRequested)
{
step.ExecutionContext.Debug($"Skip Re-evaluate condition on runner shutdown.");
}
else
{
try
{
var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(conditionReTestTraceWriter);
var condition = new BasicExpressionToken(null, null, null, step.Condition);
conditionReTestResult = templateEvaluator.EvaluateStepIf(condition, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, step.ExecutionContext.ToExpressionState());
}
catch (Exception ex)
{
// Cancel the step since we get exception while re-evaluate step condition.
Trace.Info("Caught exception from expression when re-test condition on job cancellation.");
step.ExecutionContext.Error(ex);
}
}
if (!conditionReTestResult)
{
// Cancel the step.
Trace.Info("Cancel current running step.");
step.ExecutionContext.CancelToken();
}
});
}
else
{
if (jobContext.Result != TaskResult.Canceled)
{
// mark job as cancelled
jobContext.Result = TaskResult.Canceled;
jobContext.JobContext.Status = jobContext.Result?.ToActionResult();
}
}
// Evaluate condition.
step.ExecutionContext.Debug($"Evaluating condition for step: '{step.DisplayName}'");
var conditionTraceWriter = new ConditionTraceWriter(Trace, step.ExecutionContext);
var conditionResult = false;
var conditionEvaluateError = default(Exception);
if (HostContext.RunnerShutdownToken.IsCancellationRequested)
{
step.ExecutionContext.Debug($"Skip evaluate condition on runner shutdown.");
}
else
{
try
{
var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(conditionTraceWriter);
var condition = new BasicExpressionToken(null, null, null, step.Condition);
conditionResult = templateEvaluator.EvaluateStepIf(condition, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, step.ExecutionContext.ToExpressionState());
}
catch (Exception ex)
{
Trace.Info("Caught exception from expression.");
Trace.Error(ex);
conditionEvaluateError = ex;
}
}
// no evaluate error but condition is false
if (!conditionResult && conditionEvaluateError == null)
{
// Condition == false
Trace.Info("Skipping step due to condition evaluation.");
CompleteStep(step, TaskResult.Skipped, resultCode: conditionTraceWriter.Trace);
}
else if (conditionEvaluateError != null)
{
// fail the step since there is an evaluate error.
step.ExecutionContext.Error(conditionEvaluateError);
CompleteStep(step, TaskResult.Failed);
}
else
{
// Run the step.
await RunStepAsync(step, jobContext.CancellationToken);
CompleteStep(step);
}
}
finally
{
if (jobCancelRegister != null)
{
jobCancelRegister?.Dispose();
jobCancelRegister = null;
}
}
}
// Update the job result.
if (step.ExecutionContext.Result == TaskResult.Failed)
{
Trace.Info($"Update job result with current step result '{step.ExecutionContext.Result}'.");
jobContext.Result = TaskResultUtil.MergeTaskResults(jobContext.Result, step.ExecutionContext.Result.Value);
jobContext.JobContext.Status = jobContext.Result?.ToActionResult();
}
else
{
Trace.Info($"No need for updating job result with current step result '{step.ExecutionContext.Result}'.");
}
Trace.Info($"Current state: job state = '{jobContext.Result}'");
}
}
private async Task <TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, AgentJobRequestMessage message, TaskResult?taskResult = null)
{
// Clean TEMP.
_tempDirectoryManager?.CleanupTempDirectory(jobContext);
jobContext.Section(StringUtil.Loc("StepFinishing", message.JobName));
TaskResult result = jobContext.Complete(taskResult);
try
{
await ShutdownQueue(throwOnFailure : true);
}
catch (Exception ex)
{
Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(_jobServerQueue.ShutdownAsync)}");
Trace.Error("This indicate a failure during publish output variables. Fail the job to prevent unexpected job outputs.");
Trace.Error(ex);
result = TaskResultUtil.MergeTaskResults(result, TaskResult.Failed);
}
if (!jobContext.Features.HasFlag(PlanFeatures.JobCompletedPlanEvent))
{
Trace.Info($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}");
return(result);
}
Trace.Info("Raising job completed event.");
var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result);
var completeJobRetryLimit = 5;
var exceptions = new List <Exception>();
while (completeJobRetryLimit-- > 0)
{
try
{
await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, default(CancellationToken));
return(result);
}
catch (TaskOrchestrationPlanNotFoundException ex)
{
Trace.Error($"TaskOrchestrationPlanNotFoundException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
Trace.Error(ex);
return(TaskResult.Failed);
}
catch (TaskOrchestrationPlanSecurityException ex)
{
Trace.Error($"TaskOrchestrationPlanSecurityException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
Trace.Error(ex);
return(TaskResult.Failed);
}
catch (Exception ex)
{
Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}.");
Trace.Error(ex);
exceptions.Add(ex);
}
// delay 5 seconds before next retry.
await Task.Delay(TimeSpan.FromSeconds(5));
}
// rethrow exceptions from all attempts.
throw new AggregateException(exceptions);
}
private async Task RunStepAsync(IExecutionContext jobContext, IStep step)
{
// Run the step.
Trace.Info("Starting the step.");
step.ExecutionContext.Section(StringUtil.Loc("StepStarting", step.DisplayName));
if (step.Timeout != null)
{
step.ExecutionContext.SetTimeout(timeout: step.Timeout.Value);
}
List <OperationCanceledException> allCancelExceptions = new List <OperationCanceledException>();
try
{
await step.RunAsync();
}
catch (OperationCanceledException ex)
{
if (step.ExecutionContext.CancellationToken.IsCancellationRequested &&
!jobContext.CancellationToken.IsCancellationRequested)
{
Trace.Error($"Caught timeout exception from step: {ex.Message}");
step.ExecutionContext.Error(StringUtil.Loc("StepTimedOut"));
step.ExecutionContext.Result = TaskResult.Failed;
}
else
{
// Log the exception and cancel the step.
Trace.Error($"Caught cancellation exception from step: {ex}");
step.ExecutionContext.Error(ex);
step.ExecutionContext.Result = TaskResult.Canceled;
}
//save the OperationCanceledException, merge with OperationCanceledException throw from Async Commands.
allCancelExceptions.Add(ex);
}
catch (Exception ex)
{
// Log the error and fail the step.
Trace.Error($"Caught exception from step: {ex}");
step.ExecutionContext.Error(ex);
step.ExecutionContext.Result = TaskResult.Failed;
}
// Wait till all async commands finish.
foreach (var command in step.ExecutionContext.AsyncCommands ?? new List <IAsyncCommandContext>())
{
try
{
// wait async command to finish.
await command.WaitAsync();
}
catch (OperationCanceledException ex)
{
if (step.ExecutionContext.CancellationToken.IsCancellationRequested &&
!jobContext.CancellationToken.IsCancellationRequested)
{
// Log the timeout error, set step result to falied if the current result is not canceled.
Trace.Error($"Caught timeout exception from async command {command.Name}: {ex}");
step.ExecutionContext.Error(StringUtil.Loc("StepTimedOut"));
// if the step already canceled, don't set it to failed.
step.ExecutionContext.CommandResult = TaskResultUtil.MergeTaskResults(step.ExecutionContext.CommandResult, TaskResult.Failed);
}
else
{
// log and save the OperationCanceledException, set step result to canceled if the current result is not failed.
Trace.Error($"Caught cancellation exception from async command {command.Name}: {ex}");
step.ExecutionContext.Error(ex);
// if the step already failed, don't set it to canceled.
step.ExecutionContext.CommandResult = TaskResultUtil.MergeTaskResults(step.ExecutionContext.CommandResult, TaskResult.Canceled);
}
allCancelExceptions.Add(ex);
}
catch (Exception ex)
{
// Log the error, set step result to falied if the current result is not canceled.
Trace.Error($"Caught exception from async command {command.Name}: {ex}");
step.ExecutionContext.Error(ex);
// if the step already canceled, don't set it to failed.
step.ExecutionContext.CommandResult = TaskResultUtil.MergeTaskResults(step.ExecutionContext.CommandResult, TaskResult.Failed);
}
}
// Merge executioncontext result with command result
if (step.ExecutionContext.CommandResult != null)
{
step.ExecutionContext.Result = TaskResultUtil.MergeTaskResults(step.ExecutionContext.Result, step.ExecutionContext.CommandResult.Value);
}
// TODO: consider use token.IsCancellationRequest determine step cancelled instead of catch OperationCancelException all over the place
if (step.ExecutionContext.Result == TaskResult.Canceled && allCancelExceptions.Count > 0)
{
step.ExecutionContext.Complete();
throw allCancelExceptions.First();
}
// Fixup the step result if ContinueOnError.
if (step.ExecutionContext.Result == TaskResult.Failed && step.ContinueOnError)
{
step.ExecutionContext.Result = TaskResult.SucceededWithIssues;
Trace.Info($"Updated step result: {step.ExecutionContext.Result}");
}
else
{
Trace.Info($"Step result: {step.ExecutionContext.Result}");
}
// Complete the step context.
step.ExecutionContext.Section(StringUtil.Loc("StepFinishing", step.DisplayName));
step.ExecutionContext.Complete();
}