// download all required tasks.
// make sure all task's condition inputs are valid.
// build up three list of steps for jobrunner. (pre-job, job, post-job)
public async Task <JobInitializeResult> InitializeJob(IExecutionContext jobContext, AgentJobRequestMessage message)
{
Trace.Entering();
ArgUtil.NotNull(jobContext, nameof(jobContext));
ArgUtil.NotNull(message, nameof(message));
// create a new timeline record node for 'Initialize job'
IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("InitializeJob"));
JobInitializeResult initResult = new JobInitializeResult();
using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
{
try
{
context.Start();
context.Section(StringUtil.Loc("StepStarting", StringUtil.Loc("InitializeJob")));
// Give job extension a chance to initalize
Trace.Info($"Run initial step from extension {this.GetType().Name}.");
InitializeJobExtension(context);
// Download tasks if not already in the cache
Trace.Info("Downloading task definitions.");
var taskManager = HostContext.GetService <ITaskManager>();
await taskManager.DownloadAsync(context, message.Tasks);
// Parse all Task conditions.
Trace.Info("Parsing all task's condition inputs.");
var expression = HostContext.GetService <IExpressionManager>();
Dictionary <Guid, INode> taskConditionMap = new Dictionary <Guid, INode>();
foreach (var task in message.Tasks)
{
INode condition;
if (!string.IsNullOrEmpty(task.Condition))
{
context.Debug($"Task '{task.DisplayName}' has following condition: '{task.Condition}'.");
condition = expression.Parse(context, task.Condition);
}
else if (task.AlwaysRun)
{
condition = ExpressionManager.SucceededOrFailed;
}
else
{
condition = ExpressionManager.Succeeded;
}
taskConditionMap[task.InstanceId] = condition;
}
#if OS_WINDOWS
// This is for internal testing and is not publicly supported. This will be removed from the agent at a later time.
var prepareScript = Environment.GetEnvironmentVariable("VSTS_AGENT_INIT_INTERNAL_TEMP_HACK");
if (!string.IsNullOrEmpty(prepareScript))
{
var prepareStep = new ManagementScriptStep(
scriptPath: prepareScript,
condition: ExpressionManager.Succeeded,
displayName: "Agent Initialization");
Trace.Verbose($"Adding agent init script step.");
prepareStep.Initialize(HostContext);
prepareStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), prepareStep.DisplayName);
prepareStep.AccessToken = message.Environment.SystemConnection.Authorization.Parameters["AccessToken"];
initResult.PreJobSteps.Add(prepareStep);
}
#endif
// Add pre-job steps from Tasks
Trace.Info("Adding pre-job steps from tasks.");
initResult.PreJobSteps.AddRange(taskManager.GetTasksPreJobSteps(jobContext, message.Tasks, taskConditionMap));
// Add pre-job step from Extension
Trace.Info("Adding pre-job step from extension.");
var extensionPreJobStep = GetExtensionPreJobStep(jobContext);
if (extensionPreJobStep != null)
{
initResult.PreJobSteps.Add(extensionPreJobStep);
}
// Add execution steps from Tasks
Trace.Info("Adding tasks.");
initResult.JobSteps.AddRange(taskManager.GetTasksMainSteps(jobContext, message.Tasks, taskConditionMap));
// Add post-job steps from Tasks
Trace.Info("Adding post-job steps from tasks.");
initResult.PostJobStep.AddRange(taskManager.GetTasksPostJobSteps(jobContext, message.Tasks, taskConditionMap));
// Add post-job step from Extension
Trace.Info("Adding post-job step from extension.");
var extensionPostJobStep = GetExtensionPostJobStep(jobContext);
if (extensionPostJobStep != null)
{
initResult.PostJobStep.Add(extensionPostJobStep);
}
#if OS_WINDOWS
// Add script post steps.
// This is for internal testing and is not publicly supported. This will be removed from the agent at a later time.
var finallyScript = Environment.GetEnvironmentVariable("VSTS_AGENT_CLEANUP_INTERNAL_TEMP_HACK");
if (!string.IsNullOrEmpty(finallyScript))
{
var finallyStep = new ManagementScriptStep(
scriptPath: finallyScript,
condition: ExpressionManager.Always,
displayName: "Agent Cleanup");
Trace.Verbose($"Adding agent cleanup script step.");
finallyStep.Initialize(HostContext);
finallyStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), finallyStep.DisplayName);
finallyStep.AccessToken = message.Environment.SystemConnection.Authorization.Parameters["AccessToken"];
initResult.PostJobStep.Add(finallyStep);
}
#endif
return(initResult);
}
catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
{
// Log the exception and cancel the JobExtension Initialization.
Trace.Error($"Caught cancellation exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Canceled;
throw;
}
catch (Exception ex)
{
// Log the error and fail the JobExtension Initialization.
Trace.Error($"Caught exception from JobExtension Initialization: {ex}");
context.Error(ex);
context.Result = TaskResult.Failed;
throw;
}
finally
{
context.Section(StringUtil.Loc("StepFinishing", StringUtil.Loc("InitializeJob")));
context.Complete();
}
}
}
private async Task RunAsync(AgentJobRequestMessage message, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken)
{
if (previousJobDispatch != null)
{
Trace.Verbose($"Make sure the previous job request {previousJobDispatch.JobId} has successfully finished on worker.");
await EnsureDispatchFinished(previousJobDispatch);
}
else
{
Trace.Verbose($"This is the first job request.");
}
var term = HostContext.GetService <ITerminal>();
term.WriteLine(StringUtil.Loc("RunningJob", DateTime.UtcNow, message.JobName));
// first job request renew succeed.
TaskCompletionSource <int> firstJobRequestRenewed = new TaskCompletionSource <int>();
var notification = HostContext.GetService <IJobNotification>();
// lock renew cancellation token.
using (var lockRenewalTokenSource = new CancellationTokenSource())
using (var workerProcessCancelTokenSource = new CancellationTokenSource())
{
await notification.JobStarted(message.JobId);
long requestId = message.RequestId;
Guid lockToken = message.LockToken;
// start renew job request
Trace.Info("Start renew job request.");
Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, firstJobRequestRenewed, lockRenewalTokenSource.Token);
// wait till first renew succeed or job request is canceled
// not even start worker if the first renew fail
await Task.WhenAny(firstJobRequestRenewed.Task, renewJobRequest, Task.Delay(-1, jobRequestCancellationToken));
if (renewJobRequest.IsCompleted)
{
// renew job request task complete means we run out of retry for the first job request renew.
// TODO: not need to return anything.
Trace.Info("Unable to renew job request for the first time, stop dispatching job to worker.");
return;
}
if (jobRequestCancellationToken.IsCancellationRequested)
{
await CompleteJobRequestAsync(_poolId, message, lockToken, TaskResult.Canceled);
return;
}
Task <int> workerProcessTask = null;
using (var processChannel = HostContext.CreateService <IProcessChannel>())
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
// Start the process channel.
// It's OK if StartServer bubbles an execption after the worker process has already started.
// The worker will shutdown after 30 seconds if it hasn't received the job message.
processChannel.StartServer(
// Delegate to start the child process.
startProcess: (string pipeHandleOut, string pipeHandleIn) =>
{
// Validate args.
ArgUtil.NotNullOrEmpty(pipeHandleOut, nameof(pipeHandleOut));
ArgUtil.NotNullOrEmpty(pipeHandleIn, nameof(pipeHandleIn));
// Start the child process.
var assemblyDirectory = IOUtil.GetBinPath();
string workerFileName = Path.Combine(assemblyDirectory, _workerProcessName);
workerProcessTask = processInvoker.ExecuteAsync(
workingDirectory: assemblyDirectory,
fileName: workerFileName,
arguments: "spawnclient " + pipeHandleOut + " " + pipeHandleIn,
environment: null,
cancellationToken: workerProcessCancelTokenSource.Token);
});
// Send the job request message.
// Kill the worker process if sending the job message times out. The worker
// process may have successfully received the job message.
try
{
Trace.Info("Send job request message to worker.");
using (var csSendJobRequest = new CancellationTokenSource(ChannelTimeout))
{
await processChannel.SendAsync(
messageType : MessageType.NewJobRequest,
body : JsonUtility.ToString(message),
cancellationToken : csSendJobRequest.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
// timeout 45 sec. kill worker.
Trace.Info("Job request message sending been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
// worker process been killed.
}
Trace.Info("Stop renew job request.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// not finish the job request since the job haven't run on worker at all, we will not going to set a result to server.
return;
}
TaskResult resultOnAbandonOrCancel = TaskResult.Succeeded;
// wait for renewlock, worker process or cancellation token been fired.
var completedTask = await Task.WhenAny(renewJobRequest, workerProcessTask, Task.Delay(-1, jobRequestCancellationToken));
if (completedTask == workerProcessTask)
{
// worker finished successfully, complete job request with result, stop renew lock, job has finished.
int returnCode = await workerProcessTask;
Trace.Info("Worker finished. Code: " + returnCode);
TaskResult result = TaskResultUtil.TranslateFromReturnCode(returnCode);
Trace.Info($"finish job request with result: {result}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobName, result));
// complete job request
await CompleteJobRequestAsync(_poolId, message, lockToken, result);
Trace.Info("Stop renew job request.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
return;
}
else if (completedTask == renewJobRequest)
{
resultOnAbandonOrCancel = TaskResult.Abandoned;
}
else
{
resultOnAbandonOrCancel = TaskResult.Canceled;
}
// renew job request completed or job request cancellation token been fired for RunAsync(jobrequestmessage)
// cancel worker gracefully first, then kill it after 45 sec
try
{
Trace.Info("Send job cancellation message to worker.");
using (var csSendCancel = new CancellationTokenSource(ChannelTimeout))
{
await processChannel.SendAsync(
messageType : MessageType.CancelRequest,
body : string.Empty,
cancellationToken : csSendCancel.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
Trace.Info("Job cancel message sending been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
// worker process been killed.
}
}
// wait worker to exit within 45 sec, then kill worker.
using (var csKillWorker = new CancellationTokenSource(TimeSpan.FromSeconds(45)))
{
completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, csKillWorker.Token));
}
// worker haven't exit within 45 sec.
if (completedTask != workerProcessTask)
{
Trace.Info("worker process haven't exit after 45 sec, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
// worker process been killed.
}
}
Trace.Info($"finish job request with result: {resultOnAbandonOrCancel}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobName, resultOnAbandonOrCancel));
// complete job request with cancel result, stop renew lock, job has finished.
//TODO: don't finish job request on abandon
await CompleteJobRequestAsync(_poolId, message, lockToken, resultOnAbandonOrCancel);
Trace.Info("Stop renew job request.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
}
}
}
public async Task UploadDiagnosticLogsAsync(IExecutionContext executionContext,
AgentJobRequestMessage message,
DateTime jobStartTimeUtc)
{
executionContext.Debug("Starting diagnostic file upload.");
// Setup folders
// \_layout\_work\_temp\[jobname-support]
executionContext.Debug("Setting up diagnostic log folders.");
string tempDirectory = executionContext.Variables.Agent_TempDirectory;
ArgUtil.Directory(tempDirectory, nameof(tempDirectory));
string supportRootFolder = Path.Combine(tempDirectory, message.JobName + "-support"); // TODO: Is JobName safe to use as a path? We could just generate a GUID as the name of our scoped folder?
Directory.CreateDirectory(supportRootFolder);
// \_layout\_work\_temp\[jobname-support]\files
executionContext.Debug("Creating diagnostic log files folder.");
string supportFilesFolder = Path.Combine(supportRootFolder, "files");
Directory.CreateDirectory(supportFilesFolder);
// Create the environment file
// \_layout\_work\_temp\[jobname-support]\files\environment.txt
var configurationStore = HostContext.GetService <IConfigurationStore>();
AgentSettings settings = configurationStore.GetSettings();
int agentId = settings.AgentId;
string agentName = settings.AgentName;
int poolId = settings.PoolId;
executionContext.Debug("Creating diagnostic log environment file.");
string environmentFile = Path.Combine(supportFilesFolder, "environment.txt");
#if OS_WINDOWS
string content = await GetEnvironmentContent(agentId, agentName, message.Tasks);
#else
string content = GetEnvironmentContent(agentId, agentName, message.Tasks);
#endif
File.WriteAllText(environmentFile, content);
// Create the capabilities file
var capabilitiesManager = HostContext.GetService <ICapabilitiesManager>();
Dictionary <string, string> capabilities = await capabilitiesManager.GetCapabilitiesAsync(configurationStore.GetSettings(), default(CancellationToken));
executionContext.Debug("Creating capabilities file.");
string capabilitiesFile = Path.Combine(supportFilesFolder, "capabilities.txt");
string capabilitiesContent = GetCapabilitiesContent(capabilities);
File.WriteAllText(capabilitiesFile, capabilitiesContent);
// Copy worker diag log files
List <string> workerDiagLogFiles = GetWorkerDiagLogFiles(HostContext.GetDirectory(WellKnownDirectory.Diag), jobStartTimeUtc);
executionContext.Debug($"Copying {workerDiagLogFiles.Count()} worker diag logs.");
foreach (string workerLogFile in workerDiagLogFiles)
{
ArgUtil.File(workerLogFile, nameof(workerLogFile));
string destination = Path.Combine(supportFilesFolder, Path.GetFileName(workerLogFile));
File.Copy(workerLogFile, destination);
}
executionContext.Debug("Zipping diagnostic files.");
string buildNumber = executionContext.Variables.Build_Number ?? "UnknownBuildNumber";
string buildName = $"Build {buildNumber}";
string phaseName = executionContext.Variables.System_PhaseDisplayName ?? "UnknownPhaseName";
// zip the files
string diagnosticsZipFileName = $"{buildName}-{phaseName}.zip";
string diagnosticsZipFilePath = Path.Combine(supportRootFolder, diagnosticsZipFileName);
ZipFile.CreateFromDirectory(supportFilesFolder, diagnosticsZipFilePath);
// upload the json metadata file
executionContext.Debug("Uploading diagnostic metadata file.");
string metadataFileName = $"diagnostics-{buildName}-{phaseName}.json";
string metadataFilePath = Path.Combine(supportFilesFolder, metadataFileName);
string phaseResult = GetTaskResultAsString(executionContext.Result);
IOUtil.SaveObject(new DiagnosticLogMetadata(agentName, agentId, poolId, phaseName, diagnosticsZipFileName, phaseResult), metadataFilePath);
executionContext.QueueAttachFile(type: CoreAttachmentType.DiagnosticLog, name: metadataFileName, filePath: metadataFilePath);
executionContext.QueueAttachFile(type: CoreAttachmentType.DiagnosticLog, name: diagnosticsZipFileName, filePath: diagnosticsZipFilePath);
executionContext.Debug("Diagnostic file upload complete.");
}
public async Task <TaskResult> RunAsync(AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Environment, nameof(message.Environment));
ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables));
ArgUtil.NotNull(message.Tasks, nameof(message.Tasks));
Trace.Info("Job ID {0}", message.JobId);
DateTime jobStartTimeUtc = DateTime.UtcNow;
// Agent.RunMode
RunMode runMode;
if (message.Environment.Variables.ContainsKey(Constants.Variables.Agent.RunMode) &&
Enum.TryParse(message.Environment.Variables[Constants.Variables.Agent.RunMode], ignoreCase: true, result: out runMode) &&
runMode == RunMode.Local)
{
HostContext.RunMode = runMode;
}
// System.AccessToken
if (message.Environment.Variables.ContainsKey(Constants.Variables.System.EnableAccessToken) &&
StringUtil.ConvertToBoolean(message.Environment.Variables[Constants.Variables.System.EnableAccessToken]))
{
// TODO: get access token use Util Method
message.Environment.Variables[Constants.Variables.System.AccessToken] = message.Environment.SystemConnection.Authorization.Parameters["AccessToken"];
}
// Make sure SystemConnection Url and Endpoint Url match Config Url base
ReplaceConfigUriBaseInJobRequestMessage(message);
// Setup the job server and job server queue.
var jobServer = HostContext.GetService <IJobServer>();
VssCredentials jobServerCredential = ApiUtil.GetVssCredential(message.Environment.SystemConnection);
Uri jobServerUrl = message.Environment.SystemConnection.Url;
Trace.Info($"Creating job server with URL: {jobServerUrl}");
// jobServerQueue is the throttling reporter.
_jobServerQueue = HostContext.GetService <IJobServerQueue>();
VssConnection jobConnection = ApiUtil.CreateConnection(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(_jobServerQueue) });
await jobServer.ConnectAsync(jobConnection);
_jobServerQueue.Start(message);
IExecutionContext jobContext = null;
CancellationTokenRegistration?agentShutdownRegistration = null;
try
{
// Create the job execution context.
jobContext = HostContext.CreateService <IExecutionContext>();
jobContext.InitializeJob(message, jobRequestCancellationToken);
Trace.Info("Starting the job execution context.");
jobContext.Start();
jobContext.Section(StringUtil.Loc("StepStarting", message.JobName));
agentShutdownRegistration = HostContext.AgentShutdownToken.Register(() =>
{
// log an issue, then agent get shutdown by Ctrl-C or Ctrl-Break.
// the server will use Ctrl-Break to tells the agent that operating system is shutting down.
string errorMessage;
switch (HostContext.AgentShutdownReason)
{
case ShutdownReason.UserCancelled:
errorMessage = StringUtil.Loc("UserShutdownAgent");
break;
case ShutdownReason.OperatingSystemShutdown:
errorMessage = StringUtil.Loc("OperatingSystemShutdown", Environment.MachineName);
break;
default:
throw new ArgumentException(HostContext.AgentShutdownReason.ToString(), nameof(HostContext.AgentShutdownReason));
}
jobContext.AddIssue(new Issue()
{
Type = IssueType.Error, Message = errorMessage
});
});
// Set agent version variable.
jobContext.Variables.Set(Constants.Variables.Agent.Version, Constants.Agent.Version);
jobContext.Output(StringUtil.Loc("AgentVersion", Constants.Agent.Version));
// Print proxy setting information for better diagnostic experience
var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress))
{
jobContext.Output(StringUtil.Loc("AgentRunningBehindProxy", agentWebProxy.ProxyAddress));
}
// Validate directory permissions.
string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
Trace.Info($"Validating directory permissions for: '{workDirectory}'");
try
{
Directory.CreateDirectory(workDirectory);
IOUtil.ValidateExecutePermission(workDirectory);
}
catch (Exception ex)
{
Trace.Error(ex);
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
// Set agent variables.
AgentSettings settings = HostContext.GetService <IConfigurationStore>().GetSettings();
jobContext.Variables.Set(Constants.Variables.Agent.Id, settings.AgentId.ToString(CultureInfo.InvariantCulture));
jobContext.Variables.Set(Constants.Variables.Agent.HomeDirectory, HostContext.GetDirectory(WellKnownDirectory.Root));
jobContext.Variables.Set(Constants.Variables.Agent.JobName, message.JobName);
jobContext.Variables.Set(Constants.Variables.Agent.MachineName, Environment.MachineName);
jobContext.Variables.Set(Constants.Variables.Agent.Name, settings.AgentName);
jobContext.Variables.Set(Constants.Variables.Agent.OS, VarUtil.OS);
jobContext.Variables.Set(Constants.Variables.Agent.RootDirectory, IOUtil.GetWorkPath(HostContext));
#if OS_WINDOWS
jobContext.Variables.Set(Constants.Variables.Agent.ServerOMDirectory, Path.Combine(IOUtil.GetExternalsPath(), Constants.Path.ServerOMDirectory));
#endif
jobContext.Variables.Set(Constants.Variables.Agent.WorkFolder, IOUtil.GetWorkPath(HostContext));
jobContext.Variables.Set(Constants.Variables.System.WorkFolder, IOUtil.GetWorkPath(HostContext));
if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_TOOLSDIRECTORY")))
{
string toolsDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), Constants.Path.ToolDirectory);
Directory.CreateDirectory(toolsDirectory);
jobContext.Variables.Set(Constants.Variables.Agent.ToolsDirectory, toolsDirectory);
}
// Setup TEMP directories
_tempDirectoryManager = HostContext.GetService <ITempDirectoryManager>();
_tempDirectoryManager.InitializeTempDirectory(jobContext);
// todo: task server can throw. try/catch and fail job gracefully.
// prefer task definitions url, then TFS collection url, then TFS account url
var taskServer = HostContext.GetService <ITaskServer>();
Uri taskServerUri = null;
if (!string.IsNullOrEmpty(jobContext.Variables.System_TaskDefinitionsUri))
{
taskServerUri = new Uri(jobContext.Variables.System_TaskDefinitionsUri);
}
else if (!string.IsNullOrEmpty(jobContext.Variables.System_TFCollectionUrl))
{
taskServerUri = new Uri(jobContext.Variables.System_TFCollectionUrl);
}
var taskServerCredential = ApiUtil.GetVssCredential(message.Environment.SystemConnection);
if (taskServerUri != null)
{
Trace.Info($"Creating task server with {taskServerUri}");
await taskServer.ConnectAsync(ApiUtil.CreateConnection(taskServerUri, taskServerCredential));
}
if (taskServerUri == null || !await taskServer.TaskDefinitionEndpointExist())
{
Trace.Info($"Can't determine task download url from JobMessage or the endpoint doesn't exist.");
var configStore = HostContext.GetService <IConfigurationStore>();
taskServerUri = new Uri(configStore.GetSettings().ServerUrl);
Trace.Info($"Recreate task server with configuration server url: {taskServerUri}");
await taskServer.ConnectAsync(ApiUtil.CreateConnection(taskServerUri, taskServerCredential));
}
// Expand the endpoint data values.
foreach (ServiceEndpoint endpoint in jobContext.Endpoints)
{
jobContext.Variables.ExpandValues(target: endpoint.Data);
VarUtil.ExpandEnvironmentVariables(HostContext, target: endpoint.Data);
}
// Get the job extension.
Trace.Info("Getting job extension.");
var hostType = jobContext.Variables.System_HostType;
var extensionManager = HostContext.GetService <IExtensionManager>();
// We should always have one job extension
IJobExtension jobExtension =
(extensionManager.GetExtensions <IJobExtension>() ?? new List <IJobExtension>())
.Where(x => x.HostType.HasFlag(hostType))
.FirstOrDefault();
ArgUtil.NotNull(jobExtension, nameof(jobExtension));
List <IStep> preJobSteps = new List <IStep>();
List <IStep> jobSteps = new List <IStep>();
List <IStep> postJobSteps = new List <IStep>();
try
{
Trace.Info("Initialize job. Getting all job steps.");
var initializeResult = await jobExtension.InitializeJob(jobContext, message);
preJobSteps = initializeResult.PreJobSteps;
jobSteps = initializeResult.JobSteps;
postJobSteps = initializeResult.PostJobStep;
}
catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
{
// set the job to canceled
// don't log error issue to job ExecutionContext, since server owns the job level issue
Trace.Error($"Job is canceled during initialize.");
Trace.Error($"Caught exception: {ex}");
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled));
}
catch (Exception ex)
{
// set the job to failed.
// don't log error issue to job ExecutionContext, since server owns the job level issue
Trace.Error($"Job initialize failed.");
Trace.Error($"Caught exception from {nameof(jobExtension.InitializeJob)}: {ex}");
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
// trace out all steps
Trace.Info($"Total pre-job steps: {preJobSteps.Count}.");
Trace.Verbose($"Pre-job steps: '{string.Join(", ", preJobSteps.Select(x => x.DisplayName))}'");
Trace.Info($"Total job steps: {jobSteps.Count}.");
Trace.Verbose($"Job steps: '{string.Join(", ", jobSteps.Select(x => x.DisplayName))}'");
Trace.Info($"Total post-job steps: {postJobSteps.Count}.");
Trace.Verbose($"Post-job steps: '{string.Join(", ", postJobSteps.Select(x => x.DisplayName))}'");
bool processCleanup = jobContext.Variables.GetBoolean("process.clean") ?? true;
HashSet <string> existingProcesses = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
string processLookupId = null;
if (processCleanup)
{
processLookupId = $"vsts_{Guid.NewGuid()}";
// Set the VSTS_PROCESS_LOOKUP_ID env variable.
jobContext.SetVariable(Constants.ProcessLookupId, processLookupId, false, false);
// Take a snapshot of current running processes
Dictionary <int, Process> processes = SnapshotProcesses();
foreach (var proc in processes)
{
// Pid_ProcessName
existingProcesses.Add($"{proc.Key}_{proc.Value.ProcessName}");
}
}
// Run all pre job steps
// All pre job steps are critical to the job
// Stop execution on any step failure or cancelled
Trace.Info("Run all pre-job steps.");
try
{
var stepsRunner = HostContext.GetService <IStepsRunner>();
try
{
await stepsRunner.RunAsync(jobContext, preJobSteps, JobRunStage.PreJob);
}
catch (Exception ex)
{
// StepRunner should never throw exception out.
// End up here mean there is a bug in StepRunner
// Log the error and fail the job.
Trace.Error($"Caught exception from pre-job steps {nameof(StepsRunner)}: {ex}");
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
Trace.Info($"Job result after all pre-job steps finish: {jobContext.Result}");
// Base on the Job result after all pre-job steps finish.
// Run all job steps only if the job result is still Succeeded or SucceededWithIssues
if (jobContext.Result == null ||
jobContext.Result == TaskResult.Succeeded ||
jobContext.Result == TaskResult.SucceededWithIssues)
{
Trace.Info("Run all job steps.");
try
{
await stepsRunner.RunAsync(jobContext, jobSteps, JobRunStage.Main);
}
catch (Exception ex)
{
// StepRunner should never throw exception out.
// End up here mean there is a bug in StepRunner
// Log the error and fail the job.
Trace.Error($"Caught exception from job steps {nameof(StepsRunner)}: {ex}");
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
}
else
{
Trace.Info("Skip all job steps due to pre-job step failure.");
foreach (var step in jobSteps)
{
step.ExecutionContext.Start();
step.ExecutionContext.Complete(TaskResult.Skipped);
}
}
Trace.Info($"Job result after all job steps finish: {jobContext.Result}");
// Always run all post job steps
// step might not run base on it's own condition.
Trace.Info("Run all post-job steps.");
try
{
await stepsRunner.RunAsync(jobContext, postJobSteps, JobRunStage.PostJob);
}
catch (Exception ex)
{
// StepRunner should never throw exception out.
// End up here mean there is a bug in StepRunner
// Log the error and fail the job.
Trace.Error($"Caught exception from post-job steps {nameof(StepsRunner)}: {ex}");
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
}
finally
{
if (processCleanup)
{
// Only check environment variable for any process that doesn't run before we invoke our process.
Dictionary <int, Process> currentProcesses = SnapshotProcesses();
foreach (var proc in currentProcesses)
{
if (existingProcesses.Contains($"{proc.Key}_{proc.Value.ProcessName}"))
{
Trace.Verbose($"Skip existing process. PID: {proc.Key} ({proc.Value.ProcessName})");
}
else
{
Trace.Info($"Inspecting process environment variables. PID: {proc.Key} ({proc.Value.ProcessName})");
Dictionary <string, string> env = new Dictionary <string, string>();
try
{
env = proc.Value.GetEnvironmentVariables();
foreach (var e in env)
{
Trace.Verbose($"PID:{proc.Key} ({e.Key}={e.Value})");
}
}
catch (Exception ex)
{
Trace.Verbose("Ignore any exception during read process environment variables.");
Trace.Verbose(ex.ToString());
}
if (env.TryGetValue(Constants.ProcessLookupId, out string lookupId) &&
lookupId.Equals(processLookupId, StringComparison.OrdinalIgnoreCase))
{
Trace.Info($"Terminate orphan process: pid ({proc.Key}) ({proc.Value.ProcessName})");
try
{
proc.Value.Kill();
}
catch (Exception ex)
{
Trace.Error("Catch exception during orphan process cleanup.");
Trace.Error(ex);
}
}
}
}
}
}
Trace.Info($"Job result after all post-job steps finish: {jobContext.Result}");
if (jobContext.Variables.GetBoolean(Constants.Variables.Agent.Diagnostic) ?? false)
{
Trace.Info("Support log upload starting.");
IDiagnosticLogManager diagnosticLogManager = HostContext.GetService <IDiagnosticLogManager>();
try
{
await diagnosticLogManager.UploadDiagnosticLogsAsync(executionContext : jobContext, message : message, jobStartTimeUtc : jobStartTimeUtc);
Trace.Info("Support log upload complete.");
}
catch (Exception ex)
{
// Log the error but make sure we continue gracefully.
Trace.Info("Error uploading support logs.");
Trace.Error(ex);
}
}
Trace.Info("Completing the job execution context.");
return(await CompleteJobAsync(jobServer, jobContext, message));
}
finally
{
if (agentShutdownRegistration != null)
{
agentShutdownRegistration.Value.Dispose();
agentShutdownRegistration = null;
}
await ShutdownQueue(throwOnFailure : false);
}
}
private async Task <TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, AgentJobRequestMessage message, TaskResult?taskResult = null)
{
jobContext.Section(StringUtil.Loc("StepFinishing", message.JobName));
TaskResult result = jobContext.Complete(taskResult);
try
{
await ShutdownQueue(throwOnFailure : true);
}
catch (Exception ex)
{
Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(_jobServerQueue.ShutdownAsync)}");
Trace.Error("This indicate a failure during publish output variables. Fail the job to prevent unexpected job outputs.");
Trace.Error(ex);
result = TaskResultUtil.MergeTaskResults(result, TaskResult.Failed);
}
// Clean TEMP after finish process jobserverqueue, since there might be a pending fileupload still use the TEMP dir.
_tempDirectoryManager?.CleanupTempDirectory();
if (!jobContext.Features.HasFlag(PlanFeatures.JobCompletedPlanEvent))
{
Trace.Info($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}");
return(result);
}
Trace.Info("Raising job completed event.");
var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result);
var completeJobRetryLimit = 5;
var exceptions = new List <Exception>();
while (completeJobRetryLimit-- > 0)
{
try
{
await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, default(CancellationToken));
return(result);
}
catch (TaskOrchestrationPlanNotFoundException ex)
{
Trace.Error($"TaskOrchestrationPlanNotFoundException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
Trace.Error(ex);
return(TaskResult.Failed);
}
catch (TaskOrchestrationPlanSecurityException ex)
{
Trace.Error($"TaskOrchestrationPlanSecurityException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
Trace.Error(ex);
return(TaskResult.Failed);
}
catch (Exception ex)
{
Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}.");
Trace.Error(ex);
exceptions.Add(ex);
}
// delay 5 seconds before next retry.
await Task.Delay(TimeSpan.FromSeconds(5));
}
// rethrow exceptions from all attempts.
throw new AggregateException(exceptions);
}
public TaskResult GetLocalRunJobResult(AgentJobRequestMessage message)
{
return(_localRunJobResult.Value[message.RequestId]);
}
private TestHostContext CreateTestContext([CallerMemberName] String testName = "")
{
var hc = new TestHostContext(this, testName);
_jobEc = new Agent.Worker.ExecutionContext();
_config = new Mock <IConfigurationStore>();
_extensions = new Mock <IExtensionManager>();
_jobExtension = new Mock <IJobExtension>();
_jobServer = new Mock <IJobServer>();
_jobServerQueue = new Mock <IJobServerQueue>();
_proxyConfig = new Mock <IProxyConfiguration>();
_taskServer = new Mock <ITaskServer>();
_stepRunner = new Mock <IStepsRunner>();
_logger = new Mock <IPagingLogger>();
_temp = new Mock <ITempDirectoryManager>();
if (_tokenSource != null)
{
_tokenSource.Dispose();
_tokenSource = null;
}
_tokenSource = new CancellationTokenSource();
var expressionManager = new ExpressionManager();
expressionManager.Initialize(hc);
hc.SetSingleton <IExpressionManager>(expressionManager);
_jobRunner = new JobRunner();
_jobRunner.Initialize(hc);
TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
TimelineReference timeline = new Timeline(Guid.NewGuid());
JobEnvironment environment = new JobEnvironment();
environment.Variables[Constants.Variables.System.Culture] = "en-US";
environment.SystemConnection = new ServiceEndpoint()
{
Url = new Uri("https://test.visualstudio.com"),
Authorization = new EndpointAuthorization()
{
Scheme = "Test",
}
};
environment.SystemConnection.Authorization.Parameters["AccessToken"] = "token";
List <TaskInstance> tasks = new List <TaskInstance>();
Guid JobId = Guid.NewGuid();
_message = new AgentJobRequestMessage(plan, timeline, JobId, testName, environment, tasks);
_extensions.Setup(x => x.GetExtensions <IJobExtension>()).
Returns(new[] { _jobExtension.Object }.ToList());
_initResult.PreJobSteps.Clear();
_initResult.JobSteps.Clear();
_initResult.PostJobStep.Clear();
_jobExtension.Setup(x => x.InitializeJob(It.IsAny <IExecutionContext>(), It.IsAny <AgentJobRequestMessage>())).
Returns(Task.FromResult(_initResult));
_jobExtension.Setup(x => x.HostType)
.Returns <string>(null);
_proxyConfig.Setup(x => x.ProxyUrl)
.Returns(string.Empty);
var settings = new AgentSettings
{
AgentId = 1,
AgentName = "agent1",
ServerUrl = "https://test.visualstudio.com",
WorkFolder = "_work",
};
_config.Setup(x => x.GetSettings())
.Returns(settings);
_logger.Setup(x => x.Setup(It.IsAny <Guid>(), It.IsAny <Guid>()));
hc.SetSingleton(_config.Object);
hc.SetSingleton(_jobServer.Object);
hc.SetSingleton(_jobServerQueue.Object);
hc.SetSingleton(_proxyConfig.Object);
hc.SetSingleton(_taskServer.Object);
hc.SetSingleton(_stepRunner.Object);
hc.SetSingleton(_extensions.Object);
hc.SetSingleton(_temp.Object);
hc.EnqueueInstance <IExecutionContext>(_jobEc);
hc.EnqueueInstance <IPagingLogger>(_logger.Object);
return(hc);
}
private async Task RunAsync(AgentJobRequestMessage message, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken, CancellationToken workerCancelTimeoutKillToken)
{
if (previousJobDispatch != null)
{
Trace.Verbose($"Make sure the previous job request {previousJobDispatch.JobId} has successfully finished on worker.");
await EnsureDispatchFinished(previousJobDispatch);
}
else
{
Trace.Verbose($"This is the first job request.");
}
var term = HostContext.GetService <ITerminal>();
term.WriteLine(StringUtil.Loc("RunningJob", DateTime.UtcNow, message.JobName));
// first job request renew succeed.
TaskCompletionSource <int> firstJobRequestRenewed = new TaskCompletionSource <int>();
var notification = HostContext.GetService <IJobNotification>();
// lock renew cancellation token.
using (var lockRenewalTokenSource = new CancellationTokenSource())
using (var workerProcessCancelTokenSource = new CancellationTokenSource())
{
long requestId = message.RequestId;
Guid lockToken = message.LockToken;
// start renew job request
Trace.Info($"Start renew job request {requestId} for job {message.JobId}.");
Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, firstJobRequestRenewed, lockRenewalTokenSource.Token);
// wait till first renew succeed or job request is canceled
// not even start worker if the first renew fail
await Task.WhenAny(firstJobRequestRenewed.Task, renewJobRequest, Task.Delay(-1, jobRequestCancellationToken));
if (renewJobRequest.IsCompleted)
{
// renew job request task complete means we run out of retry for the first job request renew.
Trace.Info($"Unable to renew job request for job {message.JobId} for the first time, stop dispatching job to worker.");
return;
}
if (jobRequestCancellationToken.IsCancellationRequested)
{
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request with result Cancelled
await CompleteJobRequestAsync(_poolId, message, lockToken, TaskResult.Canceled);
return;
}
Task <int> workerProcessTask = null;
object _outputLock = new object();
List <string> workerOutput = new List <string>();
using (var processChannel = HostContext.CreateService <IProcessChannel>())
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
// Start the process channel.
// It's OK if StartServer bubbles an execption after the worker process has already started.
// The worker will shutdown after 30 seconds if it hasn't received the job message.
processChannel.StartServer(
// Delegate to start the child process.
startProcess: (string pipeHandleOut, string pipeHandleIn) =>
{
// Validate args.
ArgUtil.NotNullOrEmpty(pipeHandleOut, nameof(pipeHandleOut));
ArgUtil.NotNullOrEmpty(pipeHandleIn, nameof(pipeHandleIn));
if (HostContext.RunMode == RunMode.Normal)
{
// Save STDOUT from worker, worker will use STDOUT report unhandle exception.
processInvoker.OutputDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stdout)
{
if (!string.IsNullOrEmpty(stdout.Data))
{
lock (_outputLock)
{
workerOutput.Add(stdout.Data);
}
}
};
// Save STDERR from worker, worker will use STDERR on crash.
processInvoker.ErrorDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stderr)
{
if (!string.IsNullOrEmpty(stderr.Data))
{
lock (_outputLock)
{
workerOutput.Add(stderr.Data);
}
}
};
}
else if (HostContext.RunMode == RunMode.Local)
{
processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) => Console.WriteLine(e.Data);
processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) => Console.WriteLine(e.Data);
}
// Start the child process.
var assemblyDirectory = HostContext.GetDirectory(WellKnownDirectory.Bin);
string workerFileName = Path.Combine(assemblyDirectory, _workerProcessName);
workerProcessTask = processInvoker.ExecuteAsync(
workingDirectory: assemblyDirectory,
fileName: workerFileName,
arguments: "spawnclient " + pipeHandleOut + " " + pipeHandleIn,
environment: null,
requireExitCodeZero: false,
outputEncoding: null,
killProcessOnCancel: true,
cancellationToken: workerProcessCancelTokenSource.Token);
});
// Send the job request message.
// Kill the worker process if sending the job message times out. The worker
// process may have successfully received the job message.
try
{
Trace.Info($"Send job request message to worker for job {message.JobId}.");
using (var csSendJobRequest = new CancellationTokenSource(_channelTimeout))
{
await processChannel.SendAsync(
messageType : MessageType.NewJobRequest,
body : JsonUtility.ToString(message),
cancellationToken : csSendJobRequest.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
// timeout 30 sec. kill worker.
Trace.Info($"Job request message sending for job {message.JobId} been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// not finish the job request since the job haven't run on worker at all, we will not going to set a result to server.
return;
}
// we get first jobrequest renew succeed and start the worker process with the job message.
// send notification to machine provisioner.
await notification.JobStarted(message.JobId);
try
{
TaskResult resultOnAbandonOrCancel = TaskResult.Succeeded;
// wait for renewlock, worker process or cancellation token been fired.
var completedTask = await Task.WhenAny(renewJobRequest, workerProcessTask, Task.Delay(-1, jobRequestCancellationToken));
if (completedTask == workerProcessTask)
{
// worker finished successfully, complete job request with result, attach unhandled exception reported by worker, stop renew lock, job has finished.
int returnCode = await workerProcessTask;
Trace.Info($"Worker finished for job {message.JobId}. Code: " + returnCode);
string detailInfo = null;
if (!TaskResultUtil.IsValidReturnCode(returnCode))
{
detailInfo = string.Join(Environment.NewLine, workerOutput);
Trace.Info($"Return code {returnCode} indicate worker encounter an unhandle exception or app crash, attach worker stdout/stderr to JobRequest result.");
}
TaskResult result = TaskResultUtil.TranslateFromReturnCode(returnCode);
Trace.Info($"finish job request for job {message.JobId} with result: {result}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobName, result));
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request
await CompleteJobRequestAsync(_poolId, message, lockToken, result, detailInfo);
// print out unhandle exception happened in worker after we complete job request.
// when we run out of disk space, report back to server has higher prority.
if (!string.IsNullOrEmpty(detailInfo))
{
Trace.Error("Unhandle exception happened in worker:");
Trace.Error(detailInfo);
}
return;
}
else if (completedTask == renewJobRequest)
{
resultOnAbandonOrCancel = TaskResult.Abandoned;
}
else
{
resultOnAbandonOrCancel = TaskResult.Canceled;
}
// renew job request completed or job request cancellation token been fired for RunAsync(jobrequestmessage)
// cancel worker gracefully first, then kill it after worker cancel timeout
try
{
Trace.Info($"Send job cancellation message to worker for job {message.JobId}.");
using (var csSendCancel = new CancellationTokenSource(_channelTimeout))
{
var messageType = MessageType.CancelRequest;
if (HostContext.AgentShutdownToken.IsCancellationRequested)
{
switch (HostContext.AgentShutdownReason)
{
case ShutdownReason.UserCancelled:
messageType = MessageType.AgentShutdown;
break;
case ShutdownReason.OperatingSystemShutdown:
messageType = MessageType.OperatingSystemShutdown;
break;
}
}
await processChannel.SendAsync(
messageType : messageType,
body : string.Empty,
cancellationToken : csSendCancel.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
Trace.Info($"Job cancel message sending for job {message.JobId} been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
}
// wait worker to exit
// if worker doesn't exit within timeout, then kill worker.
completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, workerCancelTimeoutKillToken));
// worker haven't exit within cancellation timeout.
if (completedTask != workerProcessTask)
{
Trace.Info($"worker process for job {message.JobId} haven't exit within cancellation timout, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
}
Trace.Info($"finish job request for job {message.JobId} with result: {resultOnAbandonOrCancel}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobName, resultOnAbandonOrCancel));
// complete job request with cancel result, stop renew lock, job has finished.
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request
await CompleteJobRequestAsync(_poolId, message, lockToken, resultOnAbandonOrCancel);
}
finally
{
// This should be the last thing to run so we don't notify external parties until actually finished
await notification.JobCompleted(message.JobId);
}
}
}
}
private TestHostContext CreateTestContext([CallerMemberName] String testName = "")
{
var hc = new TestHostContext(this, testName);
_jobEc = new Agent.Worker.ExecutionContext();
_taskManager = new Mock <ITaskManager>();
_jobServerQueue = new Mock <IJobServerQueue>();
_secretMasker = new Mock <ISecretMasker>();
_config = new Mock <IConfigurationStore>();
_logger = new Mock <IPagingLogger>();
_proxy = new Mock <IVstsAgentWebProxy>();
_express = new Mock <IExpressionManager>();
_containerProvider = new Mock <IContainerOperationProvider>();
TaskRunner step1 = new TaskRunner();
TaskRunner step2 = new TaskRunner();
TaskRunner step3 = new TaskRunner();
TaskRunner step4 = new TaskRunner();
TaskRunner step5 = new TaskRunner();
TaskRunner step6 = new TaskRunner();
TaskRunner step7 = new TaskRunner();
TaskRunner step8 = new TaskRunner();
TaskRunner step9 = new TaskRunner();
TaskRunner step10 = new TaskRunner();
TaskRunner step11 = new TaskRunner();
TaskRunner step12 = new TaskRunner();
_logger.Setup(x => x.Setup(It.IsAny <Guid>(), It.IsAny <Guid>()));
var settings = new AgentSettings
{
AgentId = 1,
AgentName = "agent1",
ServerUrl = "https://test.visualstudio.com",
WorkFolder = "_work",
};
_config.Setup(x => x.GetSettings())
.Returns(settings);
_proxy.Setup(x => x.ProxyAddress)
.Returns(string.Empty);
if (_tokenSource != null)
{
_tokenSource.Dispose();
_tokenSource = null;
}
_tokenSource = new CancellationTokenSource();
TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
TimelineReference timeline = new Timeline(Guid.NewGuid());
JobEnvironment environment = new JobEnvironment();
environment.Variables[Constants.Variables.System.Culture] = "en-US";
environment.SystemConnection = new ServiceEndpoint()
{
Url = new Uri("https://test.visualstudio.com"),
Authorization = new EndpointAuthorization()
{
Scheme = "Test",
}
};
environment.SystemConnection.Authorization.Parameters["AccessToken"] = "token";
List <TaskInstance> tasks = new List <TaskInstance>()
{
new TaskInstance()
{
InstanceId = Guid.NewGuid(),
DisplayName = "task1",
},
new TaskInstance()
{
InstanceId = Guid.NewGuid(),
DisplayName = "task2",
},
new TaskInstance()
{
InstanceId = Guid.NewGuid(),
DisplayName = "task3",
},
new TaskInstance()
{
InstanceId = Guid.NewGuid(),
DisplayName = "task4",
},
new TaskInstance()
{
InstanceId = Guid.NewGuid(),
DisplayName = "task5",
},
new TaskInstance()
{
InstanceId = Guid.NewGuid(),
DisplayName = "task6",
},
new TaskInstance()
{
InstanceId = Guid.NewGuid(),
DisplayName = "task7",
},
};
Guid JobId = Guid.NewGuid();
_message = new AgentJobRequestMessage(plan, timeline, JobId, testName, testName, environment, tasks);
_initResult.PreJobSteps.Clear();
_initResult.JobSteps.Clear();
_initResult.PostJobStep.Clear();
_taskManager.Setup(x => x.DownloadAsync(It.IsAny <IExecutionContext>(), It.IsAny <IEnumerable <TaskInstance> >()))
.Returns(Task.CompletedTask);
_taskManager.Setup(x => x.Load(It.Is <TaskInstance>(t => t.DisplayName == "task1")))
.Returns(new Definition()
{
Data = new DefinitionData()
{
PreJobExecution = null,
Execution = new ExecutionData(),
PostJobExecution = null,
},
});
_taskManager.Setup(x => x.Load(It.Is <TaskInstance>(t => t.DisplayName == "task2")))
.Returns(new Definition()
{
Data = new DefinitionData()
{
PreJobExecution = new ExecutionData(),
Execution = new ExecutionData(),
PostJobExecution = new ExecutionData(),
},
});
_taskManager.Setup(x => x.Load(It.Is <TaskInstance>(t => t.DisplayName == "task3")))
.Returns(new Definition()
{
Data = new DefinitionData()
{
PreJobExecution = new ExecutionData(),
Execution = null,
PostJobExecution = new ExecutionData(),
},
});
_taskManager.Setup(x => x.Load(It.Is <TaskInstance>(t => t.DisplayName == "task4")))
.Returns(new Definition()
{
Data = new DefinitionData()
{
PreJobExecution = new ExecutionData(),
Execution = null,
PostJobExecution = null,
},
});
_taskManager.Setup(x => x.Load(It.Is <TaskInstance>(t => t.DisplayName == "task5")))
.Returns(new Definition()
{
Data = new DefinitionData()
{
PreJobExecution = null,
Execution = null,
PostJobExecution = new ExecutionData(),
},
});
_taskManager.Setup(x => x.Load(It.Is <TaskInstance>(t => t.DisplayName == "task6")))
.Returns(new Definition()
{
Data = new DefinitionData()
{
PreJobExecution = new ExecutionData(),
Execution = new ExecutionData(),
PostJobExecution = null,
},
});
_taskManager.Setup(x => x.Load(It.Is <TaskInstance>(t => t.DisplayName == "task7")))
.Returns(new Definition()
{
Data = new DefinitionData()
{
PreJobExecution = null,
Execution = new ExecutionData(),
PostJobExecution = new ExecutionData(),
},
});
hc.SetSingleton(_taskManager.Object);
hc.SetSingleton(_config.Object);
hc.SetSingleton(_secretMasker.Object);
hc.SetSingleton(_jobServerQueue.Object);
hc.SetSingleton(_proxy.Object);
hc.SetSingleton(_express.Object);
hc.SetSingleton(_containerProvider.Object);
hc.EnqueueInstance <IPagingLogger>(_logger.Object); // jobcontext logger
hc.EnqueueInstance <IPagingLogger>(_logger.Object); // init step logger
hc.EnqueueInstance <IPagingLogger>(_logger.Object); // step 1
hc.EnqueueInstance <IPagingLogger>(_logger.Object);
hc.EnqueueInstance <IPagingLogger>(_logger.Object);
hc.EnqueueInstance <IPagingLogger>(_logger.Object);
hc.EnqueueInstance <IPagingLogger>(_logger.Object);
hc.EnqueueInstance <IPagingLogger>(_logger.Object);
hc.EnqueueInstance <IPagingLogger>(_logger.Object);
hc.EnqueueInstance <IPagingLogger>(_logger.Object);
hc.EnqueueInstance <IPagingLogger>(_logger.Object);
hc.EnqueueInstance <IPagingLogger>(_logger.Object);
hc.EnqueueInstance <IPagingLogger>(_logger.Object);
hc.EnqueueInstance <IPagingLogger>(_logger.Object); // step 12
hc.EnqueueInstance <ITaskRunner>(step1);
hc.EnqueueInstance <ITaskRunner>(step2);
hc.EnqueueInstance <ITaskRunner>(step3);
hc.EnqueueInstance <ITaskRunner>(step4);
hc.EnqueueInstance <ITaskRunner>(step5);
hc.EnqueueInstance <ITaskRunner>(step6);
hc.EnqueueInstance <ITaskRunner>(step7);
hc.EnqueueInstance <ITaskRunner>(step8);
hc.EnqueueInstance <ITaskRunner>(step9);
hc.EnqueueInstance <ITaskRunner>(step10);
hc.EnqueueInstance <ITaskRunner>(step11);
hc.EnqueueInstance <ITaskRunner>(step12);
_jobEc.Initialize(hc);
_jobEc.InitializeJob(_message, _tokenSource.Token);
return(hc);
}
public async Task <TaskResult> RunAsync(AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Environment, nameof(message.Environment));
ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables));
ArgUtil.NotNull(message.Tasks, nameof(message.Tasks));
Trace.Info("Job ID {0}", message.JobId);
if (message.Environment.Variables.ContainsKey(Constants.Variables.System.EnableAccessToken) &&
StringUtil.ConvertToBoolean(message.Environment.Variables[Constants.Variables.System.EnableAccessToken]))
{
// TODO: get access token use Util Method
message.Environment.Variables[Constants.Variables.System.AccessToken] = message.Environment.SystemConnection.Authorization.Parameters["AccessToken"];
}
// Make sure SystemConnection Url and Endpoint Url match Config Url base
ReplaceConfigUriBaseInJobRequestMessage(message);
// Setup the job server and job server queue.
var jobServer = HostContext.GetService <IJobServer>();
var jobServerCredential = ApiUtil.GetVssCredential(message.Environment.SystemConnection);
Uri jobServerUrl = message.Environment.SystemConnection.Url;
Trace.Info($"Creating job server with URL: {jobServerUrl}");
// jobServerQueue is the throttling reporter.
var jobServerQueue = HostContext.GetService <IJobServerQueue>();
var jobConnection = ApiUtil.CreateConnection(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(jobServerQueue) });
await jobServer.ConnectAsync(jobConnection);
jobServerQueue.Start(message);
IExecutionContext jobContext = null;
try
{
// Create the job execution context.
jobContext = HostContext.CreateService <IExecutionContext>();
jobContext.InitializeJob(message, jobRequestCancellationToken);
Trace.Info("Starting the job execution context.");
jobContext.Start();
// Set agent version into ExecutionContext's variables dictionary.
jobContext.Variables.Set(Constants.Variables.Agent.Version, Constants.Agent.Version);
// Print agent version into log for better diagnostic experience
jobContext.Output(StringUtil.Loc("AgentVersion", Constants.Agent.Version));
// Print proxy setting information for better diagnostic experience
var proxyConfig = HostContext.GetService <IProxyConfiguration>();
if (!string.IsNullOrEmpty(proxyConfig.ProxyUrl))
{
jobContext.Output(StringUtil.Loc("AgentRunningBehindProxy", proxyConfig.ProxyUrl));
}
// Validate directory permissions.
string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
Trace.Info($"Validating directory permissions for: '{workDirectory}'");
try
{
Directory.CreateDirectory(workDirectory);
IOUtil.ValidateExecutePermission(workDirectory);
}
catch (Exception ex)
{
Trace.Error(ex);
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Failed));
}
// Set agent variables.
AgentSettings settings = HostContext.GetService <IConfigurationStore>().GetSettings();
jobContext.Variables.Set(Constants.Variables.Agent.Id, settings.AgentId.ToString(CultureInfo.InvariantCulture));
jobContext.Variables.Set(Constants.Variables.Agent.HomeDirectory, IOUtil.GetRootPath());
jobContext.Variables.Set(Constants.Variables.Agent.JobName, message.JobName);
jobContext.Variables.Set(Constants.Variables.Agent.MachineName, Environment.MachineName);
jobContext.Variables.Set(Constants.Variables.Agent.Name, settings.AgentName);
jobContext.Variables.Set(Constants.Variables.Agent.RootDirectory, IOUtil.GetWorkPath(HostContext));
#if OS_WINDOWS
jobContext.Variables.Set(Constants.Variables.Agent.ServerOMDirectory, Path.Combine(IOUtil.GetExternalsPath(), Constants.Path.ServerOMDirectory));
#endif
jobContext.Variables.Set(Constants.Variables.Agent.WorkFolder, IOUtil.GetWorkPath(HostContext));
jobContext.Variables.Set(Constants.Variables.System.WorkFolder, IOUtil.GetWorkPath(HostContext));
// prefer task definitions url, then TFS collection url, then TFS account url
var taskServer = HostContext.GetService <ITaskServer>();
Uri taskServerUri = null;
if (!string.IsNullOrEmpty(jobContext.Variables.System_TaskDefinitionsUri))
{
taskServerUri = new Uri(jobContext.Variables.System_TaskDefinitionsUri);
}
else if (!string.IsNullOrEmpty(jobContext.Variables.System_TFCollectionUrl))
{
taskServerUri = new Uri(jobContext.Variables.System_TFCollectionUrl);
}
var taskServerCredential = ApiUtil.GetVssCredential(message.Environment.SystemConnection);
if (taskServerUri != null)
{
Trace.Info($"Creating task server with {taskServerUri}");
await taskServer.ConnectAsync(ApiUtil.CreateConnection(taskServerUri, taskServerCredential));
}
if (taskServerUri == null || !await taskServer.TaskDefinitionEndpointExist(jobRequestCancellationToken))
{
Trace.Info($"Can't determine task download url from JobMessage or the endpoint doesn't exist.");
var configStore = HostContext.GetService <IConfigurationStore>();
taskServerUri = new Uri(configStore.GetSettings().ServerUrl);
Trace.Info($"Recreate task server with configuration server url: {taskServerUri}");
await taskServer.ConnectAsync(ApiUtil.CreateConnection(taskServerUri, taskServerCredential));
}
// Expand the endpoint data values.
foreach (ServiceEndpoint endpoint in jobContext.Endpoints)
{
jobContext.Variables.ExpandValues(target: endpoint.Data);
VarUtil.ExpandEnvironmentVariables(HostContext, target: endpoint.Data);
}
// Get the job extensions.
Trace.Info("Getting job extensions.");
string hostType = jobContext.Variables.System_HostType;
var extensionManager = HostContext.GetService <IExtensionManager>();
IJobExtension[] extensions =
(extensionManager.GetExtensions <IJobExtension>() ?? new List <IJobExtension>())
.Where(x => string.Equals(x.HostType, hostType, StringComparison.OrdinalIgnoreCase))
.ToArray();
// Add the prepare steps.
Trace.Info("Adding job prepare extensions.");
List <IStep> steps = new List <IStep>();
foreach (IJobExtension extension in extensions)
{
if (extension.PrepareStep != null)
{
Trace.Verbose($"Adding {extension.GetType().Name}.{nameof(extension.PrepareStep)}.");
extension.PrepareStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), extension.PrepareStep.DisplayName);
steps.Add(extension.PrepareStep);
}
}
// Add the task steps.
Trace.Info("Adding tasks.");
foreach (TaskInstance taskInstance in message.Tasks)
{
Trace.Verbose($"Adding {taskInstance.DisplayName}.");
var taskRunner = HostContext.CreateService <ITaskRunner>();
taskRunner.ExecutionContext = jobContext.CreateChild(taskInstance.InstanceId, taskInstance.DisplayName);
taskRunner.TaskInstance = taskInstance;
steps.Add(taskRunner);
}
// Add the finally steps.
Trace.Info("Adding job finally extensions.");
foreach (IJobExtension extension in extensions)
{
if (extension.FinallyStep != null)
{
Trace.Verbose($"Adding {extension.GetType().Name}.{nameof(extension.FinallyStep)}.");
extension.FinallyStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), extension.FinallyStep.DisplayName);
steps.Add(extension.FinallyStep);
}
}
// Download tasks if not already in the cache
Trace.Info("Downloading task definitions.");
var taskManager = HostContext.GetService <ITaskManager>();
try
{
await taskManager.DownloadAsync(jobContext, message.Tasks);
}
catch (OperationCanceledException ex)
{
// set the job to canceled
Trace.Error($"Caught exception: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Canceled));
}
catch (Exception ex)
{
// Log the error and fail the job.
Trace.Error($"Caught exception from {nameof(TaskManager)}: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Failed));
}
// Run the steps.
var stepsRunner = HostContext.GetService <IStepsRunner>();
try
{
await stepsRunner.RunAsync(jobContext, steps);
}
catch (OperationCanceledException ex)
{
// set the job to canceled
Trace.Error($"Caught exception: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Canceled));
}
catch (Exception ex)
{
// Log the error and fail the job.
Trace.Error($"Caught exception from {nameof(StepsRunner)}: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Failed));
}
Trace.Info($"Job result: {jobContext.Result}");
// Complete the job.
Trace.Info("Completing the job execution context.");
return(jobContext.Complete());
}
finally
{
// Drain the job server queue.
if (jobServerQueue != null)
{
try
{
Trace.Info("Shutting down the job server queue.");
await jobServerQueue.ShutdownAsync();
}
catch (Exception ex)
{
Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(jobServerQueue.ShutdownAsync)}: {ex}");
}
}
}
}
public async Task RunIPCEndToEnd()
{
using (var server = new ProcessChannel())
{
AgentJobRequestMessage result = null;
TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
TimelineReference timeline = null;
JobEnvironment environment = new JobEnvironment();
List <TaskInstance> tasks = new List <TaskInstance>();
Guid JobId = Guid.NewGuid();
var jobRequest = new AgentJobRequestMessage(plan, timeline, JobId, "someJob", environment, tasks);
Process jobProcess;
server.StartServer((p1, p2) =>
{
string clientFileName = $"Test{IOUtil.ExeExtension}";
jobProcess = new Process();
jobProcess.StartInfo.FileName = clientFileName;
jobProcess.StartInfo.Arguments = "spawnclient " + p1 + " " + p2;
jobProcess.EnableRaisingEvents = true;
jobProcess.Start();
});
var cs = new CancellationTokenSource();
await server.SendAsync(MessageType.NewJobRequest, JsonUtility.ToString(jobRequest), cs.Token);
var packetReceiveTask = server.ReceiveAsync(cs.Token);
Task[] taskToWait = { packetReceiveTask, Task.Delay(30 * 1000) };
await Task.WhenAny(taskToWait);
bool timedOut = !packetReceiveTask.IsCompleted;
// Wait until response is received
if (timedOut)
{
cs.Cancel();
try
{
await packetReceiveTask;
}
catch (OperationCanceledException)
{
// Ignore OperationCanceledException and TaskCanceledException exceptions
}
catch (AggregateException errors)
{
// Ignore OperationCanceledException and TaskCanceledException exceptions
errors.Handle(e => e is OperationCanceledException);
}
}
else
{
result = JsonUtility.FromString <AgentJobRequestMessage>(packetReceiveTask.Result.Body);
}
// Wait until response is received
if (timedOut)
{
Assert.True(false, "Test timed out.");
}
else
{
Assert.True(jobRequest.JobId.Equals(result.JobId) && jobRequest.JobName.Equals(result.JobName));
}
}
}
private async Task <TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, AgentJobRequestMessage message, TaskResult?taskResult = null)
{
TaskResult result = jobContext.Complete(taskResult);
if (message.Plan.Version < Constants.OmitFinishAgentRequestRunPlanVersion)
{
Trace.Info($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}");
return(result);
}
await ShutdownQueue();
Trace.Info("Raising job completed event.");
IEnumerable <Variable> outputVariables = jobContext.Variables.GetOutputVariables();
//var webApiVariables = outputVariables.ToJobCompletedEventOutputVariables();
//var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result, webApiVariables);
var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result);
var completeJobRetryLimit = 5;
var exceptions = new List <Exception>();
while (completeJobRetryLimit-- > 0)
{
try
{
await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, default(CancellationToken));
return(result);
}
catch (TaskOrchestrationPlanNotFoundException ex)
{
Trace.Error($"TaskOrchestrationPlanNotFoundException received, while attempting to raise JobCompletedEvent for job {message.JobId}. Error: {ex}");
return(TaskResult.Failed);
}
catch (TaskOrchestrationPlanSecurityException ex)
{
Trace.Error($"TaskOrchestrationPlanSecurityException received, while attempting to raise JobCompletedEvent for job {message.JobId}. Error: {ex}");
return(TaskResult.Failed);
}
catch (Exception ex)
{
Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}.");
Trace.Error(ex);
exceptions.Add(ex);
}
// delay 5 seconds before next retry.
await Task.Delay(TimeSpan.FromSeconds(5));
}
// rethrow exceptions from all attempts.
throw new AggregateException(exceptions);
}
