public void ReplaceMacros(IHostContext context, Definition definition)
{
var handlerVariables = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
handlerVariables["currentdirectory"] = definition.Directory;
VarUtil.ExpandValues(context, source: handlerVariables, target: Inputs);
}
public JToken ExpandValues(JToken target)
{
_trace.Entering();
var source = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
foreach (Variable variable in _expanded.Values)
{
source[variable.Name] = variable.Value;
}
return(VarUtil.ExpandValues(_hostContext, source, target));
}
public void ExpandValues(IDictionary <string, string> target)
{
_trace.Entering();
var source = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
foreach (Variable variable in _expanded.Values)
{
var value = StringTranslator(variable.Value);
source[variable.Name] = value;
}
VarUtil.ExpandValues(_hostContext, source, target);
}
protected void AddPrependPathToEnvironment()
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(ExecutionContext.PrependPath, nameof(ExecutionContext.PrependPath));
if (ExecutionContext.PrependPath.Count == 0)
{
return;
}
// prepend path section
string prepend = string.Join(Path.PathSeparator.ToString(), ExecutionContext.PrependPath.Reverse <string>());
string originalPath = ExecutionContext.Variables.Get(Constants.PathVariable) ?? System.Environment.GetEnvironmentVariable(Constants.PathVariable) ?? string.Empty;
string newPath = VarUtil.PrependPath(prepend, originalPath);
AddEnvironmentVariable(Constants.PathVariable, newPath);
}
public string ExpandValue(string name, string value)
{
_trace.Entering();
var source = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
foreach (Variable variable in _expanded.Values)
{
source[variable.Name] = variable.Value;
}
var target = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase)
{
[name] = value
};
VarUtil.ExpandValues(_hostContext, source, target);
return(target[name]);
}
// Prefer variable over endpoint data when get build directory clean option.
// Prefer agent.clean.builddirectory over build.clean when use variable
// available value for build.clean or agent.clean.builddirectory:
// Delete entire build directory if build.clean=all is set.
// Recreate binaries dir if clean=binary is set.
// Recreate source dir if clean=src is set.
private BuildCleanOption GetBuildDirectoryCleanOption(IExecutionContext executionContext, WorkspaceOptions workspace)
{
BuildCleanOption?cleanOption = executionContext.Variables.Build_Clean;
if (cleanOption != null)
{
return(cleanOption.Value);
}
if (workspace == null)
{
return(BuildCleanOption.None);
}
else
{
Dictionary <string, string> workspaceClean = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
workspaceClean["clean"] = workspace.Clean;
executionContext.Variables.ExpandValues(target: workspaceClean);
VarUtil.ExpandEnvironmentVariables(HostContext, target: workspaceClean);
string expandedClean = workspaceClean["clean"];
if (string.Equals(expandedClean, PipelineConstants.WorkspaceCleanOptions.All, StringComparison.OrdinalIgnoreCase))
{
return(BuildCleanOption.All);
}
else if (string.Equals(expandedClean, PipelineConstants.WorkspaceCleanOptions.Resources, StringComparison.OrdinalIgnoreCase))
{
return(BuildCleanOption.Source);
}
else if (string.Equals(expandedClean, PipelineConstants.WorkspaceCleanOptions.Outputs, StringComparison.OrdinalIgnoreCase))
{
return(BuildCleanOption.Binary);
}
else
{
return(BuildCleanOption.None);
}
}
}
protected void AddPrependPathToEnvironment()
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(ExecutionContext.PrependPath, nameof(ExecutionContext.PrependPath));
if (ExecutionContext.PrependPath.Count == 0)
{
return;
}
// Prepend path.
string prepend = string.Join(Path.PathSeparator.ToString(), ExecutionContext.PrependPath.Reverse <string>());
string taskEnvPATH;
Environment.TryGetValue(Constants.PathVariable, out taskEnvPATH);
string originalPath = ExecutionContext.Variables.Get(Constants.PathVariable) ?? // Prefer a job variable.
taskEnvPATH ?? // Then a task-environment variable.
System.Environment.GetEnvironmentVariable(Constants.PathVariable) ?? // Then an environment variable.
string.Empty;
string newPath = VarUtil.PrependPath(prepend, originalPath);
AddEnvironmentVariable(Constants.PathVariable, newPath);
}
public async Task <TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Resources, nameof(message.Resources));
ArgUtil.NotNull(message.Variables, nameof(message.Variables));
ArgUtil.NotNull(message.Steps, nameof(message.Steps));
Trace.Info("Job ID {0}", message.JobId);
DateTime jobStartTimeUtc = DateTime.UtcNow;
// Agent.RunMode
RunMode runMode;
if (message.Variables.ContainsKey(Constants.Variables.Agent.RunMode) &&
Enum.TryParse(message.Variables[Constants.Variables.Agent.RunMode].Value, ignoreCase: true, result: out runMode) &&
runMode == RunMode.Local)
{
HostContext.RunMode = runMode;
}
ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
// System.AccessToken
if (message.Variables.ContainsKey(Constants.Variables.System.EnableAccessToken) &&
StringUtil.ConvertToBoolean(message.Variables[Constants.Variables.System.EnableAccessToken].Value))
{
message.Variables[Constants.Variables.System.AccessToken] = new VariableValue(systemConnection.Authorization.Parameters["AccessToken"], false);
}
// back compat TfsServerUrl
message.Variables[Constants.Variables.System.TFServerUrl] = systemConnection.Url.AbsoluteUri;
// Make sure SystemConnection Url and Endpoint Url match Config Url base for OnPremises server
if (!message.Variables.ContainsKey(Constants.Variables.System.ServerType))
{
if (!UrlUtil.IsHosted(systemConnection.Url.AbsoluteUri)) // TODO: remove this after TFS/RM move to M133
{
ReplaceConfigUriBaseInJobRequestMessage(message);
}
}
else if (string.Equals(message.Variables[Constants.Variables.System.ServerType]?.Value, "OnPremises", StringComparison.OrdinalIgnoreCase))
{
ReplaceConfigUriBaseInJobRequestMessage(message);
}
// Setup the job server and job server queue.
var jobServer = HostContext.GetService <IJobServer>();
VssCredentials jobServerCredential = ApiUtil.GetVssCredential(systemConnection);
Uri jobServerUrl = systemConnection.Url;
Trace.Info($"Creating job server with URL: {jobServerUrl}");
// jobServerQueue is the throttling reporter.
_jobServerQueue = HostContext.GetService <IJobServerQueue>();
VssConnection jobConnection = ApiUtil.CreateConnection(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(_jobServerQueue) });
await jobServer.ConnectAsync(jobConnection);
_jobServerQueue.Start(message);
IExecutionContext jobContext = null;
CancellationTokenRegistration?agentShutdownRegistration = null;
try
{
// Create the job execution context.
jobContext = HostContext.CreateService <IExecutionContext>();
jobContext.InitializeJob(message, jobRequestCancellationToken);
Trace.Info("Starting the job execution context.");
jobContext.Start();
jobContext.Section(StringUtil.Loc("StepStarting", message.JobDisplayName));
agentShutdownRegistration = HostContext.AgentShutdownToken.Register(() =>
{
// log an issue, then agent get shutdown by Ctrl-C or Ctrl-Break.
// the server will use Ctrl-Break to tells the agent that operating system is shutting down.
string errorMessage;
switch (HostContext.AgentShutdownReason)
{
case ShutdownReason.UserCancelled:
errorMessage = StringUtil.Loc("UserShutdownAgent");
break;
case ShutdownReason.OperatingSystemShutdown:
errorMessage = StringUtil.Loc("OperatingSystemShutdown", Environment.MachineName);
break;
default:
throw new ArgumentException(HostContext.AgentShutdownReason.ToString(), nameof(HostContext.AgentShutdownReason));
}
jobContext.AddIssue(new Issue()
{
Type = IssueType.Error, Message = errorMessage
});
});
// Set agent version variable.
jobContext.Variables.Set(Constants.Variables.Agent.Version, Constants.Agent.Version);
jobContext.Output(StringUtil.Loc("AgentVersion", Constants.Agent.Version));
// Print proxy setting information for better diagnostic experience
var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress))
{
jobContext.Output(StringUtil.Loc("AgentRunningBehindProxy", agentWebProxy.ProxyAddress));
}
// Validate directory permissions.
string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
Trace.Info($"Validating directory permissions for: '{workDirectory}'");
try
{
Directory.CreateDirectory(workDirectory);
IOUtil.ValidateExecutePermission(workDirectory);
}
catch (Exception ex)
{
Trace.Error(ex);
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
// Set agent variables.
AgentSettings settings = HostContext.GetService <IConfigurationStore>().GetSettings();
jobContext.Variables.Set(Constants.Variables.Agent.Id, settings.AgentId.ToString(CultureInfo.InvariantCulture));
jobContext.Variables.Set(Constants.Variables.Agent.HomeDirectory, HostContext.GetDirectory(WellKnownDirectory.Root));
jobContext.Variables.Set(Constants.Variables.Agent.JobName, message.JobDisplayName);
jobContext.Variables.Set(Constants.Variables.Agent.MachineName, Environment.MachineName);
jobContext.Variables.Set(Constants.Variables.Agent.Name, settings.AgentName);
jobContext.Variables.Set(Constants.Variables.Agent.OS, VarUtil.OS);
jobContext.Variables.Set(Constants.Variables.Agent.RootDirectory, IOUtil.GetWorkPath(HostContext));
#if OS_WINDOWS
jobContext.Variables.Set(Constants.Variables.Agent.ServerOMDirectory, Path.Combine(IOUtil.GetExternalsPath(), Constants.Path.ServerOMDirectory));
#endif
jobContext.Variables.Set(Constants.Variables.Agent.WorkFolder, IOUtil.GetWorkPath(HostContext));
jobContext.Variables.Set(Constants.Variables.System.WorkFolder, IOUtil.GetWorkPath(HostContext));
string toolsDirectory = Environment.GetEnvironmentVariable("AGENT_TOOLSDIRECTORY") ?? Environment.GetEnvironmentVariable(Constants.Variables.Agent.ToolsDirectory);
if (string.IsNullOrEmpty(toolsDirectory))
{
toolsDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), Constants.Path.ToolDirectory);
Directory.CreateDirectory(toolsDirectory);
}
else
{
Trace.Info($"Set tool cache directory base on environment: '{toolsDirectory}'");
Directory.CreateDirectory(toolsDirectory);
}
jobContext.Variables.Set(Constants.Variables.Agent.ToolsDirectory, toolsDirectory);
// Setup TEMP directories
_tempDirectoryManager = HostContext.GetService <ITempDirectoryManager>();
_tempDirectoryManager.InitializeTempDirectory(jobContext);
// todo: task server can throw. try/catch and fail job gracefully.
// prefer task definitions url, then TFS collection url, then TFS account url
var taskServer = HostContext.GetService <ITaskServer>();
Uri taskServerUri = null;
if (!string.IsNullOrEmpty(jobContext.Variables.System_TaskDefinitionsUri))
{
taskServerUri = new Uri(jobContext.Variables.System_TaskDefinitionsUri);
}
else if (!string.IsNullOrEmpty(jobContext.Variables.System_TFCollectionUrl))
{
taskServerUri = new Uri(jobContext.Variables.System_TFCollectionUrl);
}
var taskServerCredential = ApiUtil.GetVssCredential(systemConnection);
if (taskServerUri != null)
{
Trace.Info($"Creating task server with {taskServerUri}");
await taskServer.ConnectAsync(ApiUtil.CreateConnection(taskServerUri, taskServerCredential));
}
if (taskServerUri == null || !await taskServer.TaskDefinitionEndpointExist())
{
Trace.Info($"Can't determine task download url from JobMessage or the endpoint doesn't exist.");
var configStore = HostContext.GetService <IConfigurationStore>();
taskServerUri = new Uri(configStore.GetSettings().ServerUrl);
Trace.Info($"Recreate task server with configuration server url: {taskServerUri}");
await taskServer.ConnectAsync(ApiUtil.CreateConnection(taskServerUri, taskServerCredential));
}
// Expand the endpoint data values.
foreach (ServiceEndpoint endpoint in jobContext.Endpoints)
{
jobContext.Variables.ExpandValues(target: endpoint.Data);
VarUtil.ExpandEnvironmentVariables(HostContext, target: endpoint.Data);
}
// Get the job extension.
Trace.Info("Getting job extension.");
var hostType = jobContext.Variables.System_HostType;
var extensionManager = HostContext.GetService <IExtensionManager>();
// We should always have one job extension
IJobExtension jobExtension =
(extensionManager.GetExtensions <IJobExtension>() ?? new List <IJobExtension>())
.Where(x => x.HostType.HasFlag(hostType))
.FirstOrDefault();
ArgUtil.NotNull(jobExtension, nameof(jobExtension));
List <IStep> jobSteps = new List <IStep>();
try
{
Trace.Info("Initialize job. Getting all job steps.");
var initializeResult = await jobExtension.InitializeJob(jobContext, message);
jobSteps.AddRange(initializeResult.PreJobSteps);
jobSteps.AddRange(initializeResult.JobSteps);
jobSteps.AddRange(initializeResult.PostJobStep);
}
catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
{
// set the job to canceled
// don't log error issue to job ExecutionContext, since server owns the job level issue
Trace.Error($"Job is canceled during initialize.");
Trace.Error($"Caught exception: {ex}");
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled));
}
catch (Exception ex)
{
// set the job to failed.
// don't log error issue to job ExecutionContext, since server owns the job level issue
Trace.Error($"Job initialize failed.");
Trace.Error($"Caught exception from {nameof(jobExtension.InitializeJob)}: {ex}");
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
// trace out all steps
Trace.Info($"Total job steps: {jobSteps.Count}.");
Trace.Verbose($"Job steps: '{string.Join(", ", jobSteps.Select(x => x.DisplayName))}'");
bool processCleanup = jobContext.Variables.GetBoolean("process.clean") ?? true;
HashSet <string> existingProcesses = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
string processLookupId = null;
if (processCleanup)
{
processLookupId = $"vsts_{Guid.NewGuid()}";
// Set the VSTS_PROCESS_LOOKUP_ID env variable.
jobContext.SetVariable(Constants.ProcessLookupId, processLookupId, false, false);
// Take a snapshot of current running processes
Dictionary <int, Process> processes = SnapshotProcesses();
foreach (var proc in processes)
{
// Pid_ProcessName
existingProcesses.Add($"{proc.Key}_{proc.Value.ProcessName}");
}
}
// Run all job steps
Trace.Info("Run all job steps.");
var stepsRunner = HostContext.GetService <IStepsRunner>();
try
{
await stepsRunner.RunAsync(jobContext, jobSteps);
}
catch (Exception ex)
{
// StepRunner should never throw exception out.
// End up here mean there is a bug in StepRunner
// Log the error and fail the job.
Trace.Error($"Caught exception from job steps {nameof(StepsRunner)}: {ex}");
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
finally
{
if (processCleanup)
{
// Only check environment variable for any process that doesn't run before we invoke our process.
Dictionary <int, Process> currentProcesses = SnapshotProcesses();
foreach (var proc in currentProcesses)
{
if (existingProcesses.Contains($"{proc.Key}_{proc.Value.ProcessName}"))
{
Trace.Verbose($"Skip existing process. PID: {proc.Key} ({proc.Value.ProcessName})");
}
else
{
Trace.Info($"Inspecting process environment variables. PID: {proc.Key} ({proc.Value.ProcessName})");
Dictionary <string, string> env = new Dictionary <string, string>();
try
{
env = proc.Value.GetEnvironmentVariables();
foreach (var e in env)
{
Trace.Verbose($"PID:{proc.Key} ({e.Key}={e.Value})");
}
}
catch (Exception ex)
{
Trace.Verbose("Ignore any exception during read process environment variables.");
Trace.Verbose(ex.ToString());
}
if (env.TryGetValue(Constants.ProcessLookupId, out string lookupId) &&
lookupId.Equals(processLookupId, StringComparison.OrdinalIgnoreCase))
{
Trace.Info($"Terminate orphan process: pid ({proc.Key}) ({proc.Value.ProcessName})");
try
{
proc.Value.Kill();
}
catch (Exception ex)
{
Trace.Error("Catch exception during orphan process cleanup.");
Trace.Error(ex);
}
}
}
}
}
}
Trace.Info($"Job result after all job steps finish: {jobContext.Result ?? TaskResult.Succeeded}");
if (jobContext.Variables.GetBoolean(Constants.Variables.Agent.Diagnostic) ?? false)
{
Trace.Info("Support log upload starting.");
IDiagnosticLogManager diagnosticLogManager = HostContext.GetService <IDiagnosticLogManager>();
try
{
await diagnosticLogManager.UploadDiagnosticLogsAsync(executionContext : jobContext, message : message, jobStartTimeUtc : jobStartTimeUtc);
Trace.Info("Support log upload complete.");
}
catch (Exception ex)
{
// Log the error but make sure we continue gracefully.
Trace.Info("Error uploading support logs.");
Trace.Error(ex);
}
}
Trace.Info("Completing the job execution context.");
return(await CompleteJobAsync(jobServer, jobContext, message));
}
finally
{
if (agentShutdownRegistration != null)
{
agentShutdownRegistration.Value.Dispose();
agentShutdownRegistration = null;
}
await ShutdownQueue(throwOnFailure : false);
}
}
public async Task <TaskResult> RunAsync(AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Environment, nameof(message.Environment));
ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables));
ArgUtil.NotNull(message.Tasks, nameof(message.Tasks));
Trace.Info("Job ID {0}", message.JobId);
if (message.Environment.Variables.ContainsKey(Constants.Variables.System.EnableAccessToken) &&
StringUtil.ConvertToBoolean(message.Environment.Variables[Constants.Variables.System.EnableAccessToken]))
{
// TODO: get access token use Util Method
message.Environment.Variables[Constants.Variables.System.AccessToken] = message.Environment.SystemConnection.Authorization.Parameters["AccessToken"];
}
// Make sure SystemConnection Url and Endpoint Url match Config Url base
ReplaceConfigUriBaseInJobRequestMessage(message);
// Setup the job server and job server queue.
var jobServer = HostContext.GetService <IJobServer>();
var jobServerCredential = ApiUtil.GetVssCredential(message.Environment.SystemConnection);
Uri jobServerUrl = message.Environment.SystemConnection.Url;
Trace.Info($"Creating job server with URL: {jobServerUrl}");
// jobServerQueue is the throttling reporter.
var jobServerQueue = HostContext.GetService <IJobServerQueue>();
var jobConnection = ApiUtil.CreateConnection(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(jobServerQueue) });
await jobServer.ConnectAsync(jobConnection);
jobServerQueue.Start(message);
IExecutionContext jobContext = null;
try
{
// Create the job execution context.
jobContext = HostContext.CreateService <IExecutionContext>();
jobContext.InitializeJob(message, jobRequestCancellationToken);
Trace.Info("Starting the job execution context.");
jobContext.Start();
// Set agent version into ExecutionContext's variables dictionary.
jobContext.Variables.Set(Constants.Variables.Agent.Version, Constants.Agent.Version);
// Print agent version into log for better diagnostic experience
jobContext.Output(StringUtil.Loc("AgentVersion", Constants.Agent.Version));
// Print proxy setting information for better diagnostic experience
var proxyConfig = HostContext.GetService <IProxyConfiguration>();
if (!string.IsNullOrEmpty(proxyConfig.ProxyUrl))
{
jobContext.Output(StringUtil.Loc("AgentRunningBehindProxy", proxyConfig.ProxyUrl));
}
// Validate directory permissions.
string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
Trace.Info($"Validating directory permissions for: '{workDirectory}'");
try
{
Directory.CreateDirectory(workDirectory);
IOUtil.ValidateExecutePermission(workDirectory);
}
catch (Exception ex)
{
Trace.Error(ex);
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Failed));
}
// Set agent variables.
AgentSettings settings = HostContext.GetService <IConfigurationStore>().GetSettings();
jobContext.Variables.Set(Constants.Variables.Agent.Id, settings.AgentId.ToString(CultureInfo.InvariantCulture));
jobContext.Variables.Set(Constants.Variables.Agent.HomeDirectory, IOUtil.GetRootPath());
jobContext.Variables.Set(Constants.Variables.Agent.JobName, message.JobName);
jobContext.Variables.Set(Constants.Variables.Agent.MachineName, Environment.MachineName);
jobContext.Variables.Set(Constants.Variables.Agent.Name, settings.AgentName);
jobContext.Variables.Set(Constants.Variables.Agent.RootDirectory, IOUtil.GetWorkPath(HostContext));
#if OS_WINDOWS
jobContext.Variables.Set(Constants.Variables.Agent.ServerOMDirectory, Path.Combine(IOUtil.GetExternalsPath(), Constants.Path.ServerOMDirectory));
#endif
jobContext.Variables.Set(Constants.Variables.Agent.WorkFolder, IOUtil.GetWorkPath(HostContext));
jobContext.Variables.Set(Constants.Variables.System.WorkFolder, IOUtil.GetWorkPath(HostContext));
// prefer task definitions url, then TFS collection url, then TFS account url
var taskServer = HostContext.GetService <ITaskServer>();
Uri taskServerUri = null;
if (!string.IsNullOrEmpty(jobContext.Variables.System_TaskDefinitionsUri))
{
taskServerUri = new Uri(jobContext.Variables.System_TaskDefinitionsUri);
}
else if (!string.IsNullOrEmpty(jobContext.Variables.System_TFCollectionUrl))
{
taskServerUri = new Uri(jobContext.Variables.System_TFCollectionUrl);
}
var taskServerCredential = ApiUtil.GetVssCredential(message.Environment.SystemConnection);
if (taskServerUri != null)
{
Trace.Info($"Creating task server with {taskServerUri}");
await taskServer.ConnectAsync(ApiUtil.CreateConnection(taskServerUri, taskServerCredential));
}
if (taskServerUri == null || !await taskServer.TaskDefinitionEndpointExist(jobRequestCancellationToken))
{
Trace.Info($"Can't determine task download url from JobMessage or the endpoint doesn't exist.");
var configStore = HostContext.GetService <IConfigurationStore>();
taskServerUri = new Uri(configStore.GetSettings().ServerUrl);
Trace.Info($"Recreate task server with configuration server url: {taskServerUri}");
await taskServer.ConnectAsync(ApiUtil.CreateConnection(taskServerUri, taskServerCredential));
}
// Expand the endpoint data values.
foreach (ServiceEndpoint endpoint in jobContext.Endpoints)
{
jobContext.Variables.ExpandValues(target: endpoint.Data);
VarUtil.ExpandEnvironmentVariables(HostContext, target: endpoint.Data);
}
// Get the job extensions.
Trace.Info("Getting job extensions.");
string hostType = jobContext.Variables.System_HostType;
var extensionManager = HostContext.GetService <IExtensionManager>();
IJobExtension[] extensions =
(extensionManager.GetExtensions <IJobExtension>() ?? new List <IJobExtension>())
.Where(x => string.Equals(x.HostType, hostType, StringComparison.OrdinalIgnoreCase))
.ToArray();
// Add the prepare steps.
Trace.Info("Adding job prepare extensions.");
List <IStep> steps = new List <IStep>();
foreach (IJobExtension extension in extensions)
{
if (extension.PrepareStep != null)
{
Trace.Verbose($"Adding {extension.GetType().Name}.{nameof(extension.PrepareStep)}.");
extension.PrepareStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), extension.PrepareStep.DisplayName);
steps.Add(extension.PrepareStep);
}
}
// Add the task steps.
Trace.Info("Adding tasks.");
foreach (TaskInstance taskInstance in message.Tasks)
{
Trace.Verbose($"Adding {taskInstance.DisplayName}.");
var taskRunner = HostContext.CreateService <ITaskRunner>();
taskRunner.ExecutionContext = jobContext.CreateChild(taskInstance.InstanceId, taskInstance.DisplayName);
taskRunner.TaskInstance = taskInstance;
steps.Add(taskRunner);
}
// Add the finally steps.
Trace.Info("Adding job finally extensions.");
foreach (IJobExtension extension in extensions)
{
if (extension.FinallyStep != null)
{
Trace.Verbose($"Adding {extension.GetType().Name}.{nameof(extension.FinallyStep)}.");
extension.FinallyStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), extension.FinallyStep.DisplayName);
steps.Add(extension.FinallyStep);
}
}
// Download tasks if not already in the cache
Trace.Info("Downloading task definitions.");
var taskManager = HostContext.GetService <ITaskManager>();
try
{
await taskManager.DownloadAsync(jobContext, message.Tasks);
}
catch (OperationCanceledException ex)
{
// set the job to canceled
Trace.Error($"Caught exception: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Canceled));
}
catch (Exception ex)
{
// Log the error and fail the job.
Trace.Error($"Caught exception from {nameof(TaskManager)}: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Failed));
}
// Run the steps.
var stepsRunner = HostContext.GetService <IStepsRunner>();
try
{
await stepsRunner.RunAsync(jobContext, steps);
}
catch (OperationCanceledException ex)
{
// set the job to canceled
Trace.Error($"Caught exception: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Canceled));
}
catch (Exception ex)
{
// Log the error and fail the job.
Trace.Error($"Caught exception from {nameof(StepsRunner)}: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Failed));
}
Trace.Info($"Job result: {jobContext.Result}");
// Complete the job.
Trace.Info("Completing the job execution context.");
return(jobContext.Complete());
}
finally
{
// Drain the job server queue.
if (jobServerQueue != null)
{
try
{
Trace.Info("Shutting down the job server queue.");
await jobServerQueue.ShutdownAsync();
}
catch (Exception ex)
{
Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(jobServerQueue.ShutdownAsync)}: {ex}");
}
}
}
}
public async Task RunAsync()
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
ArgUtil.NotNull(ExecutionContext.Variables, nameof(ExecutionContext.Variables));
ArgUtil.NotNull(TaskInstance, nameof(TaskInstance));
var taskManager = HostContext.GetService <ITaskManager>();
var handlerFactory = HostContext.GetService <IHandlerFactory>();
// Set the task display name variable.
ExecutionContext.Variables.Set(Constants.Variables.Task.DisplayName, DisplayName);
// Load the task definition and choose the handler.
// TODO: Add a try catch here to give a better error message.
Definition definition = taskManager.Load(TaskInstance);
ArgUtil.NotNull(definition, nameof(definition));
// Print out task metadata
PrintTaskMetaData(definition);
ExecutionData currentExecution = null;
switch (Stage)
{
case JobRunStage.PreJob:
currentExecution = definition.Data?.PreJobExecution;
break;
case JobRunStage.Main:
currentExecution = definition.Data?.Execution;
break;
case JobRunStage.PostJob:
currentExecution = definition.Data?.PostJobExecution;
break;
}
;
if ((currentExecution?.All.Any(x => x is PowerShell3HandlerData)).Value &&
(currentExecution?.All.Any(x => x is PowerShellHandlerData && x.Platforms != null && x.Platforms.Contains("windows", StringComparer.OrdinalIgnoreCase))).Value)
{
// When task contains both PS and PS3 implementations, we will always prefer PS3 over PS regardless of the platform pinning.
Trace.Info("Ignore platform pinning for legacy PowerShell execution handler.");
var legacyPShandler = currentExecution?.All.Where(x => x is PowerShellHandlerData).FirstOrDefault();
legacyPShandler.Platforms = null;
}
HandlerData handlerData =
currentExecution?.All
.OrderBy(x => !x.PreferredOnCurrentPlatform()) // Sort true to false.
.ThenBy(x => x.Priority)
.FirstOrDefault();
if (handlerData == null)
{
throw new Exception(StringUtil.Loc("SupportedTaskHandlerNotFound"));
}
// Load the default input values from the definition.
Trace.Verbose("Loading default inputs.");
var inputs = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var input in (definition.Data?.Inputs ?? new TaskInputDefinition[0]))
{
string key = input?.Name?.Trim() ?? string.Empty;
if (!string.IsNullOrEmpty(key))
{
inputs[key] = input.DefaultValue?.Trim() ?? string.Empty;
}
}
// Merge the instance inputs.
Trace.Verbose("Loading instance inputs.");
foreach (var input in (TaskInstance.Inputs as IEnumerable <KeyValuePair <string, string> > ?? new KeyValuePair <string, string> [0]))
{
string key = input.Key?.Trim() ?? string.Empty;
if (!string.IsNullOrEmpty(key))
{
inputs[key] = input.Value?.Trim() ?? string.Empty;
}
}
// Expand the inputs.
Trace.Verbose("Expanding inputs.");
ExecutionContext.Variables.ExpandValues(target: inputs);
VarUtil.ExpandEnvironmentVariables(HostContext, target: inputs);
// Translate the server file path inputs to local paths.
foreach (var input in definition.Data?.Inputs ?? new TaskInputDefinition[0])
{
if (string.Equals(input.InputType, TaskInputType.FilePath, StringComparison.OrdinalIgnoreCase))
{
Trace.Verbose($"Translating file path input '{input.Name}': '{inputs[input.Name]}'");
inputs[input.Name] = TranslateFilePathInput(inputs[input.Name] ?? string.Empty);
Trace.Verbose($"Translated file path input '{input.Name}': '{inputs[input.Name]}'");
}
}
// Expand the handler inputs.
Trace.Verbose("Expanding handler inputs.");
VarUtil.ExpandValues(HostContext, source: inputs, target: handlerData.Inputs);
ExecutionContext.Variables.ExpandValues(target: handlerData.Inputs);
// Get each endpoint ID referenced by the task.
var endpointIds = new List <Guid>();
foreach (var input in definition.Data?.Inputs ?? new TaskInputDefinition[0])
{
if ((input.InputType ?? string.Empty).StartsWith("connectedService:", StringComparison.OrdinalIgnoreCase))
{
string inputKey = input?.Name?.Trim() ?? string.Empty;
string inputValue;
if (!string.IsNullOrEmpty(inputKey) &&
inputs.TryGetValue(inputKey, out inputValue) &&
!string.IsNullOrEmpty(inputValue))
{
foreach (string rawId in inputValue.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries))
{
Guid parsedId;
if (Guid.TryParse(rawId.Trim(), out parsedId) && parsedId != Guid.Empty)
{
endpointIds.Add(parsedId);
}
}
}
}
}
// Get the endpoints referenced by the task.
var endpoints = (ExecutionContext.Endpoints ?? new List <ServiceEndpoint>(0))
.Join(inner: endpointIds,
outerKeySelector: (ServiceEndpoint endpoint) => endpoint.Id,
innerKeySelector: (Guid endpointId) => endpointId,
resultSelector: (ServiceEndpoint endpoint, Guid endpointId) => endpoint)
.ToList();
// Add the system endpoint.
foreach (ServiceEndpoint endpoint in (ExecutionContext.Endpoints ?? new List <ServiceEndpoint>(0)))
{
if (string.Equals(endpoint.Name, ServiceEndpoints.SystemVssConnection, StringComparison.OrdinalIgnoreCase))
{
endpoints.Add(endpoint);
break;
}
}
// TODO: Add secure file only referenced by the task.
// Set output variables.
foreach (var outputVar in definition.Data?.OutputVariables ?? new OutputVariable[0])
{
if (outputVar != null && !string.IsNullOrEmpty(outputVar.Name))
{
ExecutionContext.OutputVariables.Add(outputVar.Name);
}
}
// Create the handler.
IHandler handler = handlerFactory.Create(
ExecutionContext,
endpoints,
handlerData,
inputs,
taskDirectory: definition.Directory,
filePathInputRootDirectory: TranslateFilePathInput(string.Empty));
// Run the task.
await handler.RunAsync();
}
public async Task RunAsync()
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
ArgUtil.NotNull(ExecutionContext.Variables, nameof(ExecutionContext.Variables));
ArgUtil.NotNull(TaskInstance, nameof(TaskInstance));
var taskManager = HostContext.GetService <ITaskManager>();
var handlerFactory = HostContext.GetService <IHandlerFactory>();
// Set the task display name variable.
ExecutionContext.Variables.Set(Constants.Variables.Task.DisplayName, DisplayName);
// Load the task definition and choose the handler.
// TODO: Add a try catch here to give a better error message.
Definition definition = taskManager.Load(TaskInstance);
ArgUtil.NotNull(definition, nameof(definition));
HandlerData handlerData =
definition.Data?.Execution?.All
.OrderBy(x => !x.PreferredOnCurrentPlatform()) // Sort true to false.
.ThenBy(x => x.Priority)
.FirstOrDefault();
if (handlerData == null)
{
throw new Exception(StringUtil.Loc("SupportedTaskHandlerNotFound"));
}
// Load the default input values from the definition.
Trace.Verbose("Loading default inputs.");
var inputs = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var input in (definition.Data?.Inputs ?? new TaskInputDefinition[0]))
{
string key = input?.Name?.Trim() ?? string.Empty;
if (!string.IsNullOrEmpty(key))
{
inputs[key] = input.DefaultValue?.Trim() ?? string.Empty;
}
}
// Merge the instance inputs.
Trace.Verbose("Loading instance inputs.");
foreach (var input in (TaskInstance.Inputs as IEnumerable <KeyValuePair <string, string> > ?? new KeyValuePair <string, string> [0]))
{
string key = input.Key?.Trim() ?? string.Empty;
if (!string.IsNullOrEmpty(key))
{
inputs[key] = input.Value?.Trim() ?? string.Empty;
}
}
// Expand the inputs.
Trace.Verbose("Expanding inputs.");
ExecutionContext.Variables.ExpandValues(target: inputs);
VarUtil.ExpandEnvironmentVariables(HostContext, target: inputs);
// Translate the server file path inputs to local paths.
foreach (var input in definition.Data?.Inputs ?? new TaskInputDefinition[0])
{
if (string.Equals(input.InputType, TaskInputType.FilePath, StringComparison.OrdinalIgnoreCase))
{
Trace.Verbose($"Translating file path input '{input.Name}': '{inputs[input.Name]}'");
inputs[input.Name] = TranslateFilePathInput(inputs[input.Name] ?? string.Empty);
Trace.Verbose($"Translated file path input '{input.Name}': '{inputs[input.Name]}'");
}
}
// Expand the handler inputs.
Trace.Verbose("Expanding handler inputs.");
VarUtil.ExpandValues(HostContext, source: inputs, target: handlerData.Inputs);
// Create the handler.
IHandler handler = handlerFactory.Create(
ExecutionContext,
handlerData,
inputs,
taskDirectory: definition.Directory,
filePathInputRootDirectory: TranslateFilePathInput(string.Empty));
// Run the task.
await handler.RunAsync();
}
public async Task <TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Resources, nameof(message.Resources));
ArgUtil.NotNull(message.Variables, nameof(message.Variables));
ArgUtil.NotNull(message.Steps, nameof(message.Steps));
Trace.Info("Job ID {0}", message.JobId);
DateTime jobStartTimeUtc = DateTime.UtcNow;
// Agent.RunMode
RunMode runMode;
if (message.Variables.ContainsKey(Constants.Variables.Agent.RunMode) &&
Enum.TryParse(message.Variables[Constants.Variables.Agent.RunMode].Value, ignoreCase: true, result: out runMode) &&
runMode == RunMode.Local)
{
HostContext.RunMode = runMode;
}
ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
// System.AccessToken
if (message.Variables.ContainsKey(Constants.Variables.System.EnableAccessToken) &&
StringUtil.ConvertToBoolean(message.Variables[Constants.Variables.System.EnableAccessToken].Value))
{
message.Variables[Constants.Variables.System.AccessToken] = new VariableValue(systemConnection.Authorization.Parameters["AccessToken"], false);
}
// back compat TfsServerUrl
message.Variables[Constants.Variables.System.TFServerUrl] = systemConnection.Url.AbsoluteUri;
// Make sure SystemConnection Url and Endpoint Url match Config Url base for OnPremises server
// System.ServerType will always be there after M133
if (!message.Variables.ContainsKey(Constants.Variables.System.ServerType) ||
string.Equals(message.Variables[Constants.Variables.System.ServerType]?.Value, "OnPremises", StringComparison.OrdinalIgnoreCase))
{
ReplaceConfigUriBaseInJobRequestMessage(message);
}
// Setup the job server and job server queue.
var jobServer = HostContext.GetService <IJobServer>();
VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection);
Uri jobServerUrl = systemConnection.Url;
Trace.Info($"Creating job server with URL: {jobServerUrl}");
// jobServerQueue is the throttling reporter.
_jobServerQueue = HostContext.GetService <IJobServerQueue>();
VssConnection jobConnection = VssUtil.CreateConnection(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(_jobServerQueue) });
await jobServer.ConnectAsync(jobConnection);
_jobServerQueue.Start(message);
HostContext.WritePerfCounter($"WorkerJobServerQueueStarted_{message.RequestId.ToString()}");
IExecutionContext jobContext = null;
CancellationTokenRegistration?agentShutdownRegistration = null;
try
{
// Create the job execution context.
jobContext = HostContext.CreateService <IExecutionContext>();
jobContext.InitializeJob(message, jobRequestCancellationToken);
Trace.Info("Starting the job execution context.");
jobContext.Start();
jobContext.Section(StringUtil.Loc("StepStarting", message.JobDisplayName));
agentShutdownRegistration = HostContext.AgentShutdownToken.Register(() =>
{
// log an issue, then agent get shutdown by Ctrl-C or Ctrl-Break.
// the server will use Ctrl-Break to tells the agent that operating system is shutting down.
string errorMessage;
switch (HostContext.AgentShutdownReason)
{
case ShutdownReason.UserCancelled:
errorMessage = StringUtil.Loc("UserShutdownAgent");
break;
case ShutdownReason.OperatingSystemShutdown:
errorMessage = StringUtil.Loc("OperatingSystemShutdown", Environment.MachineName);
break;
default:
throw new ArgumentException(HostContext.AgentShutdownReason.ToString(), nameof(HostContext.AgentShutdownReason));
}
jobContext.AddIssue(new Issue()
{
Type = IssueType.Error, Message = errorMessage
});
});
// Validate directory permissions.
string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
Trace.Info($"Validating directory permissions for: '{workDirectory}'");
try
{
Directory.CreateDirectory(workDirectory);
IOUtil.ValidateExecutePermission(workDirectory);
}
catch (Exception ex)
{
Trace.Error(ex);
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
// Set agent variables.
AgentSettings settings = HostContext.GetService <IConfigurationStore>().GetSettings();
jobContext.Variables.Set(Constants.Variables.Agent.Id, settings.AgentId.ToString(CultureInfo.InvariantCulture));
jobContext.SetVariable(Constants.Variables.Agent.HomeDirectory, HostContext.GetDirectory(WellKnownDirectory.Root), isFilePath: true);
jobContext.Variables.Set(Constants.Variables.Agent.JobName, message.JobDisplayName);
jobContext.Variables.Set(Constants.Variables.Agent.MachineName, Environment.MachineName);
jobContext.Variables.Set(Constants.Variables.Agent.Name, settings.AgentName);
jobContext.Variables.Set(Constants.Variables.Agent.OS, VarUtil.OS);
jobContext.Variables.Set(Constants.Variables.Agent.OSArchitecture, VarUtil.OSArchitecture);
jobContext.SetVariable(Constants.Variables.Agent.RootDirectory, HostContext.GetDirectory(WellKnownDirectory.Work), isFilePath: true);
if (PlatformUtil.RunningOnWindows)
{
jobContext.SetVariable(Constants.Variables.Agent.ServerOMDirectory, HostContext.GetDirectory(WellKnownDirectory.ServerOM), isFilePath: true);
}
if (!PlatformUtil.RunningOnWindows)
{
jobContext.Variables.Set(Constants.Variables.Agent.AcceptTeeEula, settings.AcceptTeeEula.ToString());
}
jobContext.SetVariable(Constants.Variables.Agent.WorkFolder, HostContext.GetDirectory(WellKnownDirectory.Work), isFilePath: true);
jobContext.SetVariable(Constants.Variables.System.WorkFolder, HostContext.GetDirectory(WellKnownDirectory.Work), isFilePath: true);
string toolsDirectory = HostContext.GetDirectory(WellKnownDirectory.Tools);
Directory.CreateDirectory(toolsDirectory);
jobContext.SetVariable(Constants.Variables.Agent.ToolsDirectory, toolsDirectory, isFilePath: true);
bool disableGitPrompt = jobContext.Variables.GetBoolean("VSTS_DISABLE_GIT_PROMPT") ?? StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("VSTS_DISABLE_GIT_PROMPT"), true);
if (disableGitPrompt &&
string.IsNullOrEmpty(jobContext.Variables.Get("GIT_TERMINAL_PROMPT")))
{
jobContext.SetVariable("GIT_TERMINAL_PROMPT", "0");
}
// Setup TEMP directories
_tempDirectoryManager = HostContext.GetService <ITempDirectoryManager>();
_tempDirectoryManager.InitializeTempDirectory(jobContext);
// todo: task server can throw. try/catch and fail job gracefully.
// prefer task definitions url, then TFS collection url, then TFS account url
var taskServer = HostContext.GetService <ITaskServer>();
Uri taskServerUri = null;
if (!string.IsNullOrEmpty(jobContext.Variables.System_TaskDefinitionsUri))
{
taskServerUri = new Uri(jobContext.Variables.System_TaskDefinitionsUri);
}
else if (!string.IsNullOrEmpty(jobContext.Variables.System_TFCollectionUrl))
{
taskServerUri = new Uri(jobContext.Variables.System_TFCollectionUrl);
}
var taskServerCredential = VssUtil.GetVssCredential(systemConnection);
if (taskServerUri != null)
{
Trace.Info($"Creating task server with {taskServerUri}");
await taskServer.ConnectAsync(VssUtil.CreateConnection(taskServerUri, taskServerCredential));
}
// for back compat TFS 2015 RTM/QU1, we may need to switch the task server url to agent config url
if (!string.Equals(message.Variables.GetValueOrDefault(Constants.Variables.System.ServerType)?.Value, "Hosted", StringComparison.OrdinalIgnoreCase))
{
if (taskServerUri == null || !await taskServer.TaskDefinitionEndpointExist())
{
Trace.Info($"Can't determine task download url from JobMessage or the endpoint doesn't exist.");
var configStore = HostContext.GetService <IConfigurationStore>();
taskServerUri = new Uri(configStore.GetSettings().ServerUrl);
Trace.Info($"Recreate task server with configuration server url: {taskServerUri}");
await taskServer.ConnectAsync(VssUtil.CreateConnection(taskServerUri, taskServerCredential));
}
}
// Expand the endpoint data values.
foreach (ServiceEndpoint endpoint in jobContext.Endpoints)
{
jobContext.Variables.ExpandValues(target: endpoint.Data);
VarUtil.ExpandEnvironmentVariables(HostContext, target: endpoint.Data);
}
// Expand the repository property values.
foreach (var repository in jobContext.Repositories)
{
// expand checkout option
var checkoutOptions = repository.Properties.Get <JToken>(Pipelines.RepositoryPropertyNames.CheckoutOptions);
if (checkoutOptions != null)
{
checkoutOptions = jobContext.Variables.ExpandValues(target: checkoutOptions);
checkoutOptions = VarUtil.ExpandEnvironmentVariables(HostContext, target: checkoutOptions);
repository.Properties.Set <JToken>(Pipelines.RepositoryPropertyNames.CheckoutOptions, checkoutOptions);;
}
// expand workspace mapping
var mappings = repository.Properties.Get <JToken>(Pipelines.RepositoryPropertyNames.Mappings);
if (mappings != null)
{
mappings = jobContext.Variables.ExpandValues(target: mappings);
mappings = VarUtil.ExpandEnvironmentVariables(HostContext, target: mappings);
repository.Properties.Set <JToken>(Pipelines.RepositoryPropertyNames.Mappings, mappings);
}
}
// Expand container properties
if (jobContext.Container != null)
{
this.ExpandProperties(jobContext.Container, jobContext.Variables);
}
foreach (var sidecar in jobContext.SidecarContainers)
{
this.ExpandProperties(sidecar, jobContext.Variables);
}
// Get the job extension.
Trace.Info("Getting job extension.");
var hostType = jobContext.Variables.System_HostType;
var extensionManager = HostContext.GetService <IExtensionManager>();
// We should always have one job extension
IJobExtension jobExtension =
(extensionManager.GetExtensions <IJobExtension>() ?? new List <IJobExtension>())
.Where(x => x.HostType.HasFlag(hostType))
.FirstOrDefault();
ArgUtil.NotNull(jobExtension, nameof(jobExtension));
List <IStep> jobSteps = null;
try
{
Trace.Info("Initialize job. Getting all job steps.");
jobSteps = await jobExtension.InitializeJob(jobContext, message);
}
catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
{
// set the job to canceled
// don't log error issue to job ExecutionContext, since server owns the job level issue
Trace.Error($"Job is canceled during initialize.");
Trace.Error($"Caught exception: {ex}");
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled));
}
catch (Exception ex)
{
// set the job to failed.
// don't log error issue to job ExecutionContext, since server owns the job level issue
Trace.Error($"Job initialize failed.");
Trace.Error($"Caught exception from {nameof(jobExtension.InitializeJob)}: {ex}");
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
// trace out all steps
Trace.Info($"Total job steps: {jobSteps.Count}.");
Trace.Verbose($"Job steps: '{string.Join(", ", jobSteps.Select(x => x.DisplayName))}'");
HostContext.WritePerfCounter($"WorkerJobInitialized_{message.RequestId.ToString()}");
// Run all job steps
Trace.Info("Run all job steps.");
var stepsRunner = HostContext.GetService <IStepsRunner>();
try
{
await stepsRunner.RunAsync(jobContext, jobSteps);
}
catch (Exception ex)
{
// StepRunner should never throw exception out.
// End up here mean there is a bug in StepRunner
// Log the error and fail the job.
Trace.Error($"Caught exception from job steps {nameof(StepsRunner)}: {ex}");
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
finally
{
Trace.Info("Finalize job.");
await jobExtension.FinalizeJob(jobContext);
}
Trace.Info($"Job result after all job steps finish: {jobContext.Result ?? TaskResult.Succeeded}");
if (jobContext.Variables.GetBoolean(Constants.Variables.Agent.Diagnostic) ?? false)
{
Trace.Info("Support log upload starting.");
IDiagnosticLogManager diagnosticLogManager = HostContext.GetService <IDiagnosticLogManager>();
try
{
await diagnosticLogManager.UploadDiagnosticLogsAsync(executionContext : jobContext, message : message, jobStartTimeUtc : jobStartTimeUtc);
Trace.Info("Support log upload complete.");
}
catch (Exception ex)
{
// Log the error but make sure we continue gracefully.
Trace.Info("Error uploading support logs.");
Trace.Error(ex);
}
}
Trace.Info("Completing the job execution context.");
return(await CompleteJobAsync(jobServer, jobContext, message));
}
finally
{
if (agentShutdownRegistration != null)
{
agentShutdownRegistration.Value.Dispose();
agentShutdownRegistration = null;
}
await ShutdownQueue(throwOnFailure : false);
}
}
public async Task RunAsync()
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
ArgUtil.NotNull(ExecutionContext.Variables, nameof(ExecutionContext.Variables));
ArgUtil.NotNull(Task, nameof(Task));
var taskManager = HostContext.GetService <ITaskManager>();
var handlerFactory = HostContext.GetService <IHandlerFactory>();
// Set the task id and display name variable.
using (var scope = ExecutionContext.Variables.CreateScope())
{
scope.Set(Constants.Variables.Task.DisplayName, DisplayName);
scope.Set(WellKnownDistributedTaskVariables.TaskInstanceId, Task.Id.ToString("D"));
scope.Set(WellKnownDistributedTaskVariables.TaskDisplayName, DisplayName);
scope.Set(WellKnownDistributedTaskVariables.TaskInstanceName, Task.Name);
// Load the task definition and choose the handler.
// TODO: Add a try catch here to give a better error message.
Definition definition = taskManager.Load(Task);
ArgUtil.NotNull(definition, nameof(definition));
// Verify task signatures if a fingerprint is configured for the Agent.
var configurationStore = HostContext.GetService <IConfigurationStore>();
AgentSettings settings = configurationStore.GetSettings();
SignatureVerificationMode verificationMode = SignatureVerificationMode.None;
if (settings.SignatureVerification != null)
{
verificationMode = settings.SignatureVerification.Mode;
}
if (verificationMode != SignatureVerificationMode.None)
{
ISignatureService signatureService = HostContext.CreateService <ISignatureService>();
Boolean verificationSuccessful = await signatureService.VerifyAsync(definition, ExecutionContext.CancellationToken);
if (verificationSuccessful)
{
ExecutionContext.Output(StringUtil.Loc("TaskSignatureVerificationSucceeeded"));
// Only extract if it's not the checkout task.
if (!String.IsNullOrEmpty(definition.ZipPath))
{
taskManager.Extract(ExecutionContext, Task);
}
}
else
{
String message = StringUtil.Loc("TaskSignatureVerificationFailed");
if (verificationMode == SignatureVerificationMode.Error)
{
throw new InvalidOperationException(message);
}
else
{
ExecutionContext.Warning(message);
}
}
}
// Print out task metadata
PrintTaskMetaData(definition);
ExecutionData currentExecution = null;
switch (Stage)
{
case JobRunStage.PreJob:
currentExecution = definition.Data?.PreJobExecution;
break;
case JobRunStage.Main:
currentExecution = definition.Data?.Execution;
break;
case JobRunStage.PostJob:
currentExecution = definition.Data?.PostJobExecution;
break;
}
;
HandlerData handlerData = GetHandlerData(ExecutionContext, currentExecution, PlatformUtil.HostOS);
if (handlerData == null)
{
if (PlatformUtil.RunningOnWindows)
{
throw new InvalidOperationException(StringUtil.Loc("SupportedTaskHandlerNotFoundWindows", $"{PlatformUtil.HostOS}({PlatformUtil.HostArchitecture})"));
}
throw new InvalidOperationException(StringUtil.Loc("SupportedTaskHandlerNotFoundLinux"));
}
Trace.Info($"Handler data is of type {handlerData}");
Variables runtimeVariables = ExecutionContext.Variables;
IStepHost stepHost = HostContext.CreateService <IDefaultStepHost>();
var stepTarget = ExecutionContext.StepTarget();
// Setup container stephost and the right runtime variables for running job inside container.
if (stepTarget is ContainerInfo containerTarget)
{
if (handlerData is AgentPluginHandlerData)
{
// plugin handler always runs on the Host, the runtime variables needs to the variable works on the Host, ex: file path variable System.DefaultWorkingDirectory
Dictionary <string, VariableValue> variableCopy = new Dictionary <string, VariableValue>(StringComparer.OrdinalIgnoreCase);
foreach (var publicVar in ExecutionContext.Variables.Public)
{
variableCopy[publicVar.Key] = new VariableValue(stepTarget.TranslateToHostPath(publicVar.Value));
}
foreach (var secretVar in ExecutionContext.Variables.Private)
{
variableCopy[secretVar.Key] = new VariableValue(stepTarget.TranslateToHostPath(secretVar.Value), true);
}
List <string> expansionWarnings;
runtimeVariables = new Variables(HostContext, variableCopy, out expansionWarnings);
expansionWarnings?.ForEach(x => ExecutionContext.Warning(x));
}
else if (handlerData is NodeHandlerData || handlerData is Node10HandlerData || handlerData is PowerShell3HandlerData)
{
// Only the node, node10, and powershell3 handlers support running inside container.
// Make sure required container is already created.
ArgUtil.NotNullOrEmpty(containerTarget.ContainerId, nameof(containerTarget.ContainerId));
var containerStepHost = HostContext.CreateService <IContainerStepHost>();
containerStepHost.Container = containerTarget;
stepHost = containerStepHost;
}
else
{
throw new NotSupportedException(String.Format("Task '{0}' is using legacy execution handler '{1}' which is not supported in container execution flow.", definition.Data.FriendlyName, handlerData.GetType().ToString()));
}
}
// Load the default input values from the definition.
Trace.Verbose("Loading default inputs.");
var inputs = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var input in (definition.Data?.Inputs ?? new TaskInputDefinition[0]))
{
string key = input?.Name?.Trim() ?? string.Empty;
if (!string.IsNullOrEmpty(key))
{
inputs[key] = input.DefaultValue?.Trim() ?? string.Empty;
}
}
// Merge the instance inputs.
Trace.Verbose("Loading instance inputs.");
foreach (var input in (Task.Inputs as IEnumerable <KeyValuePair <string, string> > ?? new KeyValuePair <string, string> [0]))
{
string key = input.Key?.Trim() ?? string.Empty;
if (!string.IsNullOrEmpty(key))
{
inputs[key] = input.Value?.Trim() ?? string.Empty;
}
}
// Expand the inputs.
Trace.Verbose("Expanding inputs.");
runtimeVariables.ExpandValues(target: inputs);
VarUtil.ExpandEnvironmentVariables(HostContext, target: inputs);
// Translate the server file path inputs to local paths.
foreach (var input in definition.Data?.Inputs ?? new TaskInputDefinition[0])
{
if (string.Equals(input.InputType, TaskInputType.FilePath, StringComparison.OrdinalIgnoreCase))
{
Trace.Verbose($"Translating file path input '{input.Name}': '{inputs[input.Name]}'");
inputs[input.Name] = stepHost.ResolvePathForStepHost(TranslateFilePathInput(inputs[input.Name] ?? string.Empty));
Trace.Verbose($"Translated file path input '{input.Name}': '{inputs[input.Name]}'");
}
}
// Load the task environment.
Trace.Verbose("Loading task environment.");
var environment = new Dictionary <string, string>(VarUtil.EnvironmentVariableKeyComparer);
foreach (var env in (Task.Environment ?? new Dictionary <string, string>(0)))
{
string key = env.Key?.Trim() ?? string.Empty;
if (!string.IsNullOrEmpty(key))
{
environment[key] = env.Value?.Trim() ?? string.Empty;
}
}
// Expand the inputs.
Trace.Verbose("Expanding task environment.");
runtimeVariables.ExpandValues(target: environment);
VarUtil.ExpandEnvironmentVariables(HostContext, target: environment);
// Expand the handler inputs.
Trace.Verbose("Expanding handler inputs.");
VarUtil.ExpandValues(HostContext, source: inputs, target: handlerData.Inputs);
runtimeVariables.ExpandValues(target: handlerData.Inputs);
// Get each endpoint ID referenced by the task.
var endpointIds = new List <Guid>();
foreach (var input in definition.Data?.Inputs ?? new TaskInputDefinition[0])
{
if ((input.InputType ?? string.Empty).StartsWith("connectedService:", StringComparison.OrdinalIgnoreCase))
{
string inputKey = input?.Name?.Trim() ?? string.Empty;
string inputValue;
if (!string.IsNullOrEmpty(inputKey) &&
inputs.TryGetValue(inputKey, out inputValue) &&
!string.IsNullOrEmpty(inputValue))
{
foreach (string rawId in inputValue.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries))
{
Guid parsedId;
if (Guid.TryParse(rawId.Trim(), out parsedId) && parsedId != Guid.Empty)
{
endpointIds.Add(parsedId);
}
}
}
}
}
if (endpointIds.Count > 0 &&
(runtimeVariables.GetBoolean(WellKnownDistributedTaskVariables.RestrictSecrets) ?? false) &&
(runtimeVariables.GetBoolean(Microsoft.TeamFoundation.Build.WebApi.BuildVariables.IsFork) ?? false))
{
ExecutionContext.Result = TaskResult.Skipped;
ExecutionContext.ResultCode = $"References service endpoint. PRs from repository forks are not allowed to access secrets in the pipeline. For more information see https://go.microsoft.com/fwlink/?linkid=862029 ";
return;
}
// Get the endpoints referenced by the task.
var endpoints = (ExecutionContext.Endpoints ?? new List <ServiceEndpoint>(0))
.Join(inner: endpointIds,
outerKeySelector: (ServiceEndpoint endpoint) => endpoint.Id,
innerKeySelector: (Guid endpointId) => endpointId,
resultSelector: (ServiceEndpoint endpoint, Guid endpointId) => endpoint)
.ToList();
// Add the system endpoint.
foreach (ServiceEndpoint endpoint in (ExecutionContext.Endpoints ?? new List <ServiceEndpoint>(0)))
{
if (string.Equals(endpoint.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase))
{
endpoints.Add(endpoint);
break;
}
}
// Get each secure file ID referenced by the task.
var secureFileIds = new List <Guid>();
foreach (var input in definition.Data?.Inputs ?? new TaskInputDefinition[0])
{
if (string.Equals(input.InputType ?? string.Empty, "secureFile", StringComparison.OrdinalIgnoreCase))
{
string inputKey = input?.Name?.Trim() ?? string.Empty;
string inputValue;
if (!string.IsNullOrEmpty(inputKey) &&
inputs.TryGetValue(inputKey, out inputValue) &&
!string.IsNullOrEmpty(inputValue))
{
foreach (string rawId in inputValue.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries))
{
Guid parsedId;
if (Guid.TryParse(rawId.Trim(), out parsedId) && parsedId != Guid.Empty)
{
secureFileIds.Add(parsedId);
}
}
}
}
}
if (secureFileIds.Count > 0 &&
(runtimeVariables.GetBoolean(WellKnownDistributedTaskVariables.RestrictSecrets) ?? false) &&
(runtimeVariables.GetBoolean(Microsoft.TeamFoundation.Build.WebApi.BuildVariables.IsFork) ?? false))
{
ExecutionContext.Result = TaskResult.Skipped;
ExecutionContext.ResultCode = $"References secure file. PRs from repository forks are not allowed to access secrets in the pipeline. For more information see https://go.microsoft.com/fwlink/?linkid=862029";
return;
}
// Get the endpoints referenced by the task.
var secureFiles = (ExecutionContext.SecureFiles ?? new List <SecureFile>(0))
.Join(inner: secureFileIds,
outerKeySelector: (SecureFile secureFile) => secureFile.Id,
innerKeySelector: (Guid secureFileId) => secureFileId,
resultSelector: (SecureFile secureFile, Guid secureFileId) => secureFile)
.ToList();
// Set output variables.
foreach (var outputVar in definition.Data?.OutputVariables ?? new OutputVariable[0])
{
if (outputVar != null && !string.IsNullOrEmpty(outputVar.Name))
{
ExecutionContext.OutputVariables.Add(outputVar.Name);
}
}
// translate inputs
inputs = inputs.ToDictionary(kvp => kvp.Key, kvp => ExecutionContext.TranslatePathForStepTarget(kvp.Value));
// Create the handler.
IHandler handler = handlerFactory.Create(
ExecutionContext,
Task.Reference,
stepHost,
endpoints,
secureFiles,
handlerData,
inputs,
environment,
runtimeVariables,
taskDirectory: definition.Directory);
// Run the task.
await handler.RunAsync();
}
}
public async Task RunAsync()
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
ArgUtil.NotNull(ExecutionContext.Variables, nameof(ExecutionContext.Variables));
ArgUtil.NotNull(Task, nameof(Task));
var taskManager = HostContext.GetService <ITaskManager>();
var handlerFactory = HostContext.GetService <IHandlerFactory>();
// Set the task id and display name variable.
using (var scope = ExecutionContext.Variables.CreateScope())
{
scope.Set(Constants.Variables.Task.DisplayName, DisplayName);
scope.Set(WellKnownDistributedTaskVariables.TaskInstanceId, Task.Id.ToString("D"));
scope.Set(WellKnownDistributedTaskVariables.TaskDisplayName, DisplayName);
scope.Set(WellKnownDistributedTaskVariables.TaskInstanceName, Task.Name);
// Load the task definition and choose the handler.
// TODO: Add a try catch here to give a better error message.
Definition definition = taskManager.Load(Task);
ArgUtil.NotNull(definition, nameof(definition));
// Verify Signatures and Re-Extract Tasks if neccessary
await VerifyTask(taskManager, definition);
// Print out task metadata
PrintTaskMetaData(definition);
ExecutionData currentExecution = null;
switch (Stage)
{
case JobRunStage.PreJob:
currentExecution = definition.Data?.PreJobExecution;
break;
case JobRunStage.Main:
currentExecution = definition.Data?.Execution;
break;
case JobRunStage.PostJob:
currentExecution = definition.Data?.PostJobExecution;
break;
}
;
HandlerData handlerData = GetHandlerData(ExecutionContext, currentExecution, PlatformUtil.HostOS);
if (handlerData == null)
{
if (PlatformUtil.RunningOnWindows)
{
throw new InvalidOperationException(StringUtil.Loc("SupportedTaskHandlerNotFoundWindows", $"{PlatformUtil.HostOS}({PlatformUtil.HostArchitecture})"));
}
throw new InvalidOperationException(StringUtil.Loc("SupportedTaskHandlerNotFoundLinux"));
}
Trace.Info($"Handler data is of type {handlerData}");
Variables runtimeVariables = ExecutionContext.Variables;
IStepHost stepHost = HostContext.CreateService <IDefaultStepHost>();
var stepTarget = ExecutionContext.StepTarget();
// Setup container stephost and the right runtime variables for running job inside container.
if (stepTarget is ContainerInfo containerTarget)
{
if (Stage == JobRunStage.PostJob &&
AgentKnobs.SkipPostExeceutionIfTargetContainerStopped.GetValue(ExecutionContext).AsBoolean())
{
try
{
// Check that the target container is still running, if not Skip task execution
IDockerCommandManager dockerManager = HostContext.GetService <IDockerCommandManager>();
bool isContainerRunning = await dockerManager.IsContainerRunning(ExecutionContext, containerTarget.ContainerId);
if (!isContainerRunning)
{
ExecutionContext.Result = TaskResult.Skipped;
ExecutionContext.ResultCode = $"Target container - {containerTarget.ContainerName} has been stopped, task post-execution will be skipped";
return;
}
}
catch (Exception ex)
{
ExecutionContext.Write(WellKnownTags.Warning, $"Failed to check container state for task post-execution. Exception: {ex}");
}
}
if (handlerData is AgentPluginHandlerData)
{
// plugin handler always runs on the Host, the runtime variables needs to the variable works on the Host, ex: file path variable System.DefaultWorkingDirectory
Dictionary <string, VariableValue> variableCopy = new Dictionary <string, VariableValue>(StringComparer.OrdinalIgnoreCase);
foreach (var publicVar in ExecutionContext.Variables.Public)
{
variableCopy[publicVar.Key] = new VariableValue(stepTarget.TranslateToHostPath(publicVar.Value));
}
foreach (var secretVar in ExecutionContext.Variables.Private)
{
variableCopy[secretVar.Key] = new VariableValue(stepTarget.TranslateToHostPath(secretVar.Value), true);
}
List <string> expansionWarnings;
runtimeVariables = new Variables(HostContext, variableCopy, out expansionWarnings);
expansionWarnings?.ForEach(x => ExecutionContext.Warning(x));
}
else if (handlerData is BaseNodeHandlerData || handlerData is PowerShell3HandlerData)
{
// Only the node, node10, and powershell3 handlers support running inside container.
// Make sure required container is already created.
ArgUtil.NotNullOrEmpty(containerTarget.ContainerId, nameof(containerTarget.ContainerId));
var containerStepHost = HostContext.CreateService <IContainerStepHost>();
containerStepHost.Container = containerTarget;
stepHost = containerStepHost;
}
else
{
throw new NotSupportedException(String.Format("Task '{0}' is using legacy execution handler '{1}' which is not supported in container execution flow.", definition.Data.FriendlyName, handlerData.GetType().ToString()));
}
}
// Load the default input values from the definition.
Trace.Verbose("Loading default inputs.");
var inputs = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var input in (definition.Data?.Inputs ?? new TaskInputDefinition[0]))
{
string key = input?.Name?.Trim() ?? string.Empty;
if (!string.IsNullOrEmpty(key))
{
if (AgentKnobs.DisableInputTrimming.GetValue(ExecutionContext).AsBoolean())
{
inputs[key] = input.DefaultValue ?? string.Empty;
}
else
{
inputs[key] = input.DefaultValue?.Trim() ?? string.Empty;
}
}
}
// Merge the instance inputs.
Trace.Verbose("Loading instance inputs.");
foreach (var input in (Task.Inputs as IEnumerable <KeyValuePair <string, string> > ?? new KeyValuePair <string, string> [0]))
{
string key = input.Key?.Trim() ?? string.Empty;
if (!string.IsNullOrEmpty(key))
{
if (AgentKnobs.DisableInputTrimming.GetValue(ExecutionContext).AsBoolean())
{
inputs[key] = input.Value ?? string.Empty;
}
else
{
inputs[key] = input.Value?.Trim() ?? string.Empty;
}
}
}
// Expand the inputs.
Trace.Verbose("Expanding inputs.");
runtimeVariables.ExpandValues(target: inputs);
// We need to verify inputs of the tasks that were injected by decorators, to check if they contain secrets,
// for security reasons execution of tasks in this case should be skipped.
// Target task inputs could be injected into the decorator's tasks if the decorator has post-task-tasks or pre-task-tasks targets,
// such tasks will have names that start with __system_pretargettask_ or __system_posttargettask_.
var taskDecoratorManager = HostContext.GetService <ITaskDecoratorManager>();
if (taskDecoratorManager.IsInjectedTaskForTarget(Task.Name) &&
taskDecoratorManager.IsInjectedInputsContainsSecrets(inputs, out var inputsWithSecrets))
{
var inputsForReport = taskDecoratorManager.GenerateTaskResultMessage(inputsWithSecrets);
ExecutionContext.Result = TaskResult.Skipped;
ExecutionContext.ResultCode = StringUtil.Loc("SecretsAreNotAllowedInInjectedTaskInputs", inputsForReport);
return;
}
VarUtil.ExpandEnvironmentVariables(HostContext, target: inputs);
// Translate the server file path inputs to local paths.
foreach (var input in definition.Data?.Inputs ?? new TaskInputDefinition[0])
{
if (string.Equals(input.InputType, TaskInputType.FilePath, StringComparison.OrdinalIgnoreCase))
{
Trace.Verbose($"Translating file path input '{input.Name}': '{inputs[input.Name]}'");
inputs[input.Name] = stepHost.ResolvePathForStepHost(TranslateFilePathInput(inputs[input.Name] ?? string.Empty));
Trace.Verbose($"Translated file path input '{input.Name}': '{inputs[input.Name]}'");
}
}
// Load the task environment.
Trace.Verbose("Loading task environment.");
var environment = new Dictionary <string, string>(VarUtil.EnvironmentVariableKeyComparer);
foreach (var env in (Task.Environment ?? new Dictionary <string, string>(0)))
{
string key = env.Key?.Trim() ?? string.Empty;
if (!string.IsNullOrEmpty(key))
{
environment[key] = env.Value?.Trim() ?? string.Empty;
}
}
// Expand the inputs.
Trace.Verbose("Expanding task environment.");
runtimeVariables.ExpandValues(target: environment);
VarUtil.ExpandEnvironmentVariables(HostContext, target: environment);
// Expand the handler inputs.
Trace.Verbose("Expanding handler inputs.");
VarUtil.ExpandValues(HostContext, source: inputs, target: handlerData.Inputs);
runtimeVariables.ExpandValues(target: handlerData.Inputs);
// Get each endpoint ID referenced by the task.
var endpointIds = new List <Guid>();
foreach (var input in definition.Data?.Inputs ?? new TaskInputDefinition[0])
{
if ((input.InputType ?? string.Empty).StartsWith("connectedService:", StringComparison.OrdinalIgnoreCase))
{
string inputKey = input?.Name?.Trim() ?? string.Empty;
string inputValue;
if (!string.IsNullOrEmpty(inputKey) &&
inputs.TryGetValue(inputKey, out inputValue) &&
!string.IsNullOrEmpty(inputValue))
{
foreach (string rawId in inputValue.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries))
{
Guid parsedId;
if (Guid.TryParse(rawId.Trim(), out parsedId) && parsedId != Guid.Empty)
{
endpointIds.Add(parsedId);
}
}
}
}
}
if (endpointIds.Count > 0 &&
(runtimeVariables.GetBoolean(WellKnownDistributedTaskVariables.RestrictSecrets) ?? false) &&
(runtimeVariables.GetBoolean(Microsoft.TeamFoundation.Build.WebApi.BuildVariables.IsFork) ?? false))
{
ExecutionContext.Result = TaskResult.Skipped;
ExecutionContext.ResultCode = $"References service endpoint. PRs from repository forks are not allowed to access secrets in the pipeline. For more information see https://go.microsoft.com/fwlink/?linkid=862029 ";
return;
}
// Get the endpoints referenced by the task.
var endpoints = (ExecutionContext.Endpoints ?? new List <ServiceEndpoint>(0))
.Join(inner: endpointIds,
outerKeySelector: (ServiceEndpoint endpoint) => endpoint.Id,
innerKeySelector: (Guid endpointId) => endpointId,
resultSelector: (ServiceEndpoint endpoint, Guid endpointId) => endpoint)
.ToList();
// Add the system endpoint.
foreach (ServiceEndpoint endpoint in (ExecutionContext.Endpoints ?? new List <ServiceEndpoint>(0)))
{
if (string.Equals(endpoint.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase))
{
endpoints.Add(endpoint);
break;
}
}
// Get each secure file ID referenced by the task.
var secureFileIds = new List <Guid>();
foreach (var input in definition.Data?.Inputs ?? new TaskInputDefinition[0])
{
if (string.Equals(input.InputType ?? string.Empty, "secureFile", StringComparison.OrdinalIgnoreCase))
{
string inputKey = input?.Name?.Trim() ?? string.Empty;
string inputValue;
if (!string.IsNullOrEmpty(inputKey) &&
inputs.TryGetValue(inputKey, out inputValue) &&
!string.IsNullOrEmpty(inputValue))
{
foreach (string rawId in inputValue.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries))
{
Guid parsedId;
if (Guid.TryParse(rawId.Trim(), out parsedId) && parsedId != Guid.Empty)
{
secureFileIds.Add(parsedId);
}
}
}
}
}
if (secureFileIds.Count > 0 &&
(runtimeVariables.GetBoolean(WellKnownDistributedTaskVariables.RestrictSecrets) ?? false) &&
(runtimeVariables.GetBoolean(Microsoft.TeamFoundation.Build.WebApi.BuildVariables.IsFork) ?? false))
{
ExecutionContext.Result = TaskResult.Skipped;
ExecutionContext.ResultCode = $"References secure file. PRs from repository forks are not allowed to access secrets in the pipeline. For more information see https://go.microsoft.com/fwlink/?linkid=862029";
return;
}
// Get the endpoints referenced by the task.
var secureFiles = (ExecutionContext.SecureFiles ?? new List <SecureFile>(0))
.Join(inner: secureFileIds,
outerKeySelector: (SecureFile secureFile) => secureFile.Id,
innerKeySelector: (Guid secureFileId) => secureFileId,
resultSelector: (SecureFile secureFile, Guid secureFileId) => secureFile)
.ToList();
// Set output variables.
foreach (var outputVar in definition.Data?.OutputVariables ?? new OutputVariable[0])
{
if (outputVar != null && !string.IsNullOrEmpty(outputVar.Name))
{
ExecutionContext.OutputVariables.Add(outputVar.Name);
}
}
// translate inputs
inputs = inputs.ToDictionary(kvp => kvp.Key, kvp => ExecutionContext.TranslatePathForStepTarget(kvp.Value));
// Create the handler.
IHandler handler = handlerFactory.Create(
ExecutionContext,
Task.Reference,
stepHost,
endpoints,
secureFiles,
handlerData,
inputs,
environment,
runtimeVariables,
taskDirectory: definition.Directory);
// Run the task.
int retryCount = this.Task.RetryCountOnTaskFailure;
if (retryCount > 0)
{
if (retryCount > RetryCountOnTaskFailureLimit)
{
ExecutionContext.Warning(StringUtil.Loc("RetryCountLimitExceeded", RetryCountOnTaskFailureLimit, retryCount));
retryCount = RetryCountOnTaskFailureLimit;
}
RetryHelper rh = new RetryHelper(ExecutionContext, retryCount);
await rh.RetryStep(async() => await handler.RunAsync(), RetryHelper.ExponentialDelay);
}
else
{
await handler.RunAsync();
}
}
}
public async Task RunAsync()
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
ArgUtil.NotNull(ExecutionContext.Variables, nameof(ExecutionContext.Variables));
ArgUtil.NotNull(TaskInstance, nameof(TaskInstance));
var taskManager = HostContext.GetService <ITaskManager>();
var handlerFactory = HostContext.GetService <IHandlerFactory>();
// Set the task display name variable.
ExecutionContext.Variables.Set(Constants.Variables.Task.DisplayName, DisplayName);
// Load the task definition and choose the handler.
// TODO: Add a try catch here to give a better error message.
Definition definition = taskManager.Load(TaskInstance);
ArgUtil.NotNull(definition, nameof(definition));
// Print out task metadata
PrintTaskMetaData(definition);
if ((definition.Data?.Execution?.All.Any(x => x is PowerShell3HandlerData)).Value &&
(definition.Data?.Execution?.All.Any(x => x is PowerShellHandlerData && x.Platforms != null && x.Platforms.Contains("windows", StringComparer.OrdinalIgnoreCase))).Value)
{
// When task contains both PS and PS3 implementations, we will always prefer PS3 over PS regardless of the platform pinning.
Trace.Info("Ignore platform pinning for legacy PowerShell execution handler.");
var legacyPShandler = definition.Data?.Execution?.All.Where(x => x is PowerShellHandlerData).FirstOrDefault();
legacyPShandler.Platforms = null;
}
HandlerData handlerData =
definition.Data?.Execution?.All
.OrderBy(x => !x.PreferredOnCurrentPlatform()) // Sort true to false.
.ThenBy(x => x.Priority)
.FirstOrDefault();
if (handlerData == null)
{
string[] supportedHandlers;
#if OS_WINDOWS
supportedHandlers = new string[] { "Node", "PowerShell3", "PowerShell", "AzurePowerShell", "PowerShellExe", "Process" };
#else
supportedHandlers = new string[] { "Node" };
#endif
throw new Exception(StringUtil.Loc("SupportedTaskHandlerNotFound", string.Join(", ", supportedHandlers)));
}
// Load the default input values from the definition.
Trace.Verbose("Loading default inputs.");
var inputs = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var input in (definition.Data?.Inputs ?? new TaskInputDefinition[0]))
{
string key = input?.Name?.Trim() ?? string.Empty;
if (!string.IsNullOrEmpty(key))
{
inputs[key] = input.DefaultValue?.Trim() ?? string.Empty;
}
}
// Merge the instance inputs.
Trace.Verbose("Loading instance inputs.");
foreach (var input in (TaskInstance.Inputs as IEnumerable <KeyValuePair <string, string> > ?? new KeyValuePair <string, string> [0]))
{
string key = input.Key?.Trim() ?? string.Empty;
if (!string.IsNullOrEmpty(key))
{
inputs[key] = input.Value?.Trim() ?? string.Empty;
}
}
// Expand the inputs.
Trace.Verbose("Expanding inputs.");
ExecutionContext.Variables.ExpandValues(target: inputs);
VarUtil.ExpandEnvironmentVariables(HostContext, target: inputs);
// Translate the server file path inputs to local paths.
foreach (var input in definition.Data?.Inputs ?? new TaskInputDefinition[0])
{
if (string.Equals(input.InputType, TaskInputType.FilePath, StringComparison.OrdinalIgnoreCase))
{
Trace.Verbose($"Translating file path input '{input.Name}': '{inputs[input.Name]}'");
inputs[input.Name] = TranslateFilePathInput(inputs[input.Name] ?? string.Empty);
Trace.Verbose($"Translated file path input '{input.Name}': '{inputs[input.Name]}'");
}
}
// Expand the handler inputs.
Trace.Verbose("Expanding handler inputs.");
VarUtil.ExpandValues(HostContext, source: inputs, target: handlerData.Inputs);
ExecutionContext.Variables.ExpandValues(target: handlerData.Inputs);
// Create the handler.
IHandler handler = handlerFactory.Create(
ExecutionContext,
handlerData,
inputs,
taskDirectory: definition.Directory,
filePathInputRootDirectory: TranslateFilePathInput(string.Empty));
// Run the task.
await handler.RunAsync();
}
public async Task <TaskResult> RunAsync(AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Environment, nameof(message.Environment));
ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables));
ArgUtil.NotNull(message.Tasks, nameof(message.Tasks));
Trace.Info("Job ID {0}", message.JobId);
// System.AccessToken
if (message.Environment.Variables.ContainsKey(Constants.Variables.System.EnableAccessToken) &&
StringUtil.ConvertToBoolean(message.Environment.Variables[Constants.Variables.System.EnableAccessToken]))
{
// TODO: get access token use Util Method
message.Environment.Variables[Constants.Variables.System.AccessToken] = message.Environment.SystemConnection.Authorization.Parameters["AccessToken"];
}
// Make sure SystemConnection Url and Endpoint Url match Config Url base
ReplaceConfigUriBaseInJobRequestMessage(message);
// Setup the job server and job server queue.
var jobServer = HostContext.GetService <IJobServer>();
VssCredentials jobServerCredential = ApiUtil.GetVssCredential(message.Environment.SystemConnection);
Uri jobServerUrl = message.Environment.SystemConnection.Url;
Trace.Info($"Creating job server with URL: {jobServerUrl}");
// jobServerQueue is the throttling reporter.
_jobServerQueue = HostContext.GetService <IJobServerQueue>();
VssConnection jobConnection = ApiUtil.CreateConnection(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(_jobServerQueue) });
await jobServer.ConnectAsync(jobConnection);
_jobServerQueue.Start(message);
IExecutionContext jobContext = null;
try
{
// Create the job execution context.
jobContext = HostContext.CreateService <IExecutionContext>();
jobContext.InitializeJob(message, jobRequestCancellationToken);
Trace.Info("Starting the job execution context.");
jobContext.Start();
jobContext.Section(StringUtil.Loc("StepStarting", message.JobName));
// Set agent version variable.
jobContext.Variables.Set(Constants.Variables.Agent.Version, Constants.Agent.Version);
jobContext.Output(StringUtil.Loc("AgentVersion", Constants.Agent.Version));
// Print proxy setting information for better diagnostic experience
var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress))
{
jobContext.Output(StringUtil.Loc("AgentRunningBehindProxy", agentWebProxy.ProxyAddress));
}
// Validate directory permissions.
string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
Trace.Info($"Validating directory permissions for: '{workDirectory}'");
try
{
Directory.CreateDirectory(workDirectory);
IOUtil.ValidateExecutePermission(workDirectory);
}
catch (Exception ex)
{
Trace.Error(ex);
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
// Set agent variables.
AgentSettings settings = HostContext.GetService <IConfigurationStore>().GetSettings();
jobContext.Variables.Set(Constants.Variables.Agent.Id, settings.AgentId.ToString(CultureInfo.InvariantCulture));
jobContext.Variables.Set(Constants.Variables.Agent.HomeDirectory, IOUtil.GetRootPath());
jobContext.Variables.Set(Constants.Variables.Agent.JobName, message.JobName);
jobContext.Variables.Set(Constants.Variables.Agent.MachineName, Environment.MachineName);
jobContext.Variables.Set(Constants.Variables.Agent.Name, settings.AgentName);
jobContext.Variables.Set(Constants.Variables.Agent.RootDirectory, IOUtil.GetWorkPath(HostContext));
#if OS_WINDOWS
jobContext.Variables.Set(Constants.Variables.Agent.ServerOMDirectory, Path.Combine(IOUtil.GetExternalsPath(), Constants.Path.ServerOMDirectory));
#endif
jobContext.Variables.Set(Constants.Variables.Agent.WorkFolder, IOUtil.GetWorkPath(HostContext));
jobContext.Variables.Set(Constants.Variables.System.WorkFolder, IOUtil.GetWorkPath(HostContext));
if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_TOOLSDIRECTORY")))
{
string toolsDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), Constants.Path.ToolDirectory);
Directory.CreateDirectory(toolsDirectory);
jobContext.Variables.Set(Constants.Variables.Agent.ToolsDirectory, toolsDirectory);
}
// Setup TEMP directories
_tempDirectoryManager = HostContext.GetService <ITempDirectoryManager>();
_tempDirectoryManager.InitializeTempDirectory(jobContext);
// todo: task server can throw. try/catch and fail job gracefully.
// prefer task definitions url, then TFS collection url, then TFS account url
var taskServer = HostContext.GetService <ITaskServer>();
Uri taskServerUri = null;
if (!string.IsNullOrEmpty(jobContext.Variables.System_TaskDefinitionsUri))
{
taskServerUri = new Uri(jobContext.Variables.System_TaskDefinitionsUri);
}
else if (!string.IsNullOrEmpty(jobContext.Variables.System_TFCollectionUrl))
{
taskServerUri = new Uri(jobContext.Variables.System_TFCollectionUrl);
}
var taskServerCredential = ApiUtil.GetVssCredential(message.Environment.SystemConnection);
if (taskServerUri != null)
{
Trace.Info($"Creating task server with {taskServerUri}");
await taskServer.ConnectAsync(ApiUtil.CreateConnection(taskServerUri, taskServerCredential));
}
if (taskServerUri == null || !await taskServer.TaskDefinitionEndpointExist(jobRequestCancellationToken))
{
Trace.Info($"Can't determine task download url from JobMessage or the endpoint doesn't exist.");
var configStore = HostContext.GetService <IConfigurationStore>();
taskServerUri = new Uri(configStore.GetSettings().ServerUrl);
Trace.Info($"Recreate task server with configuration server url: {taskServerUri}");
await taskServer.ConnectAsync(ApiUtil.CreateConnection(taskServerUri, taskServerCredential));
}
// Expand the endpoint data values.
foreach (ServiceEndpoint endpoint in jobContext.Endpoints)
{
jobContext.Variables.ExpandValues(target: endpoint.Data);
VarUtil.ExpandEnvironmentVariables(HostContext, target: endpoint.Data);
}
// Get the job extension.
Trace.Info("Getting job extension.");
var hostType = jobContext.Variables.System_HostType;
var extensionManager = HostContext.GetService <IExtensionManager>();
// We should always have one job extension
IJobExtension jobExtension =
(extensionManager.GetExtensions <IJobExtension>() ?? new List <IJobExtension>())
.Where(x => x.HostType.HasFlag(hostType))
.FirstOrDefault();
ArgUtil.NotNull(jobExtension, nameof(jobExtension));
List <IStep> preJobSteps = new List <IStep>();
List <IStep> jobSteps = new List <IStep>();
List <IStep> postJobSteps = new List <IStep>();
try
{
Trace.Info("Initialize job. Getting all job steps.");
var initalizeResult = await jobExtension.InitializeJob(jobContext, message);
preJobSteps = initalizeResult.PreJobSteps;
jobSteps = initalizeResult.JobSteps;
postJobSteps = initalizeResult.PostJobStep;
}
catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
{
// set the job to canceled
// don't log error issue to job ExecutionContext, since server owns the job level issue
Trace.Error($"Job is canclled during initialize.");
Trace.Error($"Caught exception: {ex}");
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled));
}
catch (Exception ex)
{
// set the job to failed.
// don't log error issue to job ExecutionContext, since server owns the job level issue
Trace.Error($"Job initialize failed.");
Trace.Error($"Caught exception from {nameof(jobExtension.InitializeJob)}: {ex}");
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
// trace out all steps
Trace.Info($"Total pre-job steps: {preJobSteps.Count}.");
Trace.Verbose($"Pre-job steps: '{string.Join(", ", preJobSteps.Select(x => x.DisplayName))}'");
Trace.Info($"Total job steps: {jobSteps.Count}.");
Trace.Verbose($"Job steps: '{string.Join(", ", jobSteps.Select(x => x.DisplayName))}'");
Trace.Info($"Total post-job steps: {postJobSteps.Count}.");
Trace.Verbose($"Post-job steps: '{string.Join(", ", postJobSteps.Select(x => x.DisplayName))}'");
// Run all pre job steps
// All pre job steps are critical to the job
// Stop exection on any step failure or cancelled
Trace.Info("Run all pre-job steps.");
var stepsRunner = HostContext.GetService <IStepsRunner>();
try
{
await stepsRunner.RunAsync(jobContext, preJobSteps, JobRunStage.PreJob);
}
catch (Exception ex)
{
// StepRunner should never throw exception out.
// End up here mean there is a bug in StepRunner
// Log the error and fail the job.
Trace.Error($"Caught exception from pre-job steps {nameof(StepsRunner)}: {ex}");
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
Trace.Info($"Job result after all pre-job steps finish: {jobContext.Result}");
// Base on the Job result after all pre-job steps finish.
// Run all job steps only if the job result is still Succeeded or SucceededWithIssues
if (jobContext.Result == null ||
jobContext.Result == TaskResult.Succeeded ||
jobContext.Result == TaskResult.SucceededWithIssues)
{
Trace.Info("Run all job steps.");
try
{
await stepsRunner.RunAsync(jobContext, jobSteps, JobRunStage.Main);
}
catch (Exception ex)
{
// StepRunner should never throw exception out.
// End up here mean there is a bug in StepRunner
// Log the error and fail the job.
Trace.Error($"Caught exception from job steps {nameof(StepsRunner)}: {ex}");
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
}
else
{
Trace.Info("Skip all job steps due to pre-job step failure.");
foreach (var step in jobSteps)
{
step.ExecutionContext.Start();
step.ExecutionContext.Complete(TaskResult.Skipped);
}
}
Trace.Info($"Job result after all job steps finish: {jobContext.Result}");
// Always run all post job steps
// step might not run base on it's own condition.
Trace.Info("Run all post-job steps.");
try
{
await stepsRunner.RunAsync(jobContext, postJobSteps, JobRunStage.PostJob);
}
catch (Exception ex)
{
// StepRunner should never throw exception out.
// End up here mean there is a bug in StepRunner
// Log the error and fail the job.
Trace.Error($"Caught exception from post-job steps {nameof(StepsRunner)}: {ex}");
jobContext.Error(ex);
return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
}
Trace.Info($"Job result after all post-job steps finish: {jobContext.Result}");
// Complete the job.
Trace.Info("Completing the job execution context.");
return(await CompleteJobAsync(jobServer, jobContext, message));
}
finally
{
await ShutdownQueue(throwOnFailure : false);
}
}
public async Task RunAsync()
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
ArgUtil.NotNull(ExecutionContext.Variables, nameof(ExecutionContext.Variables));
ArgUtil.NotNull(Task, nameof(Task));
var taskManager = HostContext.GetService <ITaskManager>();
var handlerFactory = HostContext.GetService <IHandlerFactory>();
// Set the task id and display name variable.
ExecutionContext.Variables.Set(Constants.Variables.Task.DisplayName, DisplayName);
ExecutionContext.Variables.Set(WellKnownDistributedTaskVariables.TaskInstanceId, Task.Id.ToString("D"));
ExecutionContext.Variables.Set(WellKnownDistributedTaskVariables.TaskDisplayName, DisplayName);
ExecutionContext.Variables.Set(WellKnownDistributedTaskVariables.TaskInstanceName, Task.Name);
// Load the task definition and choose the handler.
// TODO: Add a try catch here to give a better error message.
Definition definition = taskManager.Load(Task);
ArgUtil.NotNull(definition, nameof(definition));
// Print out task metadata
PrintTaskMetaData(definition);
ExecutionData currentExecution = null;
switch (Stage)
{
case JobRunStage.PreJob:
currentExecution = definition.Data?.PreJobExecution;
break;
case JobRunStage.Main:
currentExecution = definition.Data?.Execution;
break;
case JobRunStage.PostJob:
currentExecution = definition.Data?.PostJobExecution;
break;
}
;
if ((currentExecution?.All.Any(x => x is PowerShell3HandlerData)).Value &&
(currentExecution?.All.Any(x => x is PowerShellHandlerData && x.Platforms != null && x.Platforms.Contains("windows", StringComparer.OrdinalIgnoreCase))).Value)
{
// When task contains both PS and PS3 implementations, we will always prefer PS3 over PS regardless of the platform pinning.
Trace.Info("Ignore platform pinning for legacy PowerShell execution handler.");
var legacyPShandler = currentExecution?.All.Where(x => x is PowerShellHandlerData).FirstOrDefault();
legacyPShandler.Platforms = null;
}
HandlerData handlerData =
currentExecution?.All
.OrderBy(x => !x.PreferredOnCurrentPlatform()) // Sort true to false.
.ThenBy(x => x.Priority)
.FirstOrDefault();
if (handlerData == null)
{
#if OS_WINDOWS
throw new Exception(StringUtil.Loc("SupportedTaskHandlerNotFoundWindows", $"{Constants.Agent.Platform}({Constants.Agent.PlatformArchitecture})"));
#else
throw new Exception(StringUtil.Loc("SupportedTaskHandlerNotFoundLinux"));
#endif
}
Variables runtimeVariables = ExecutionContext.Variables;
IStepHost stepHost = HostContext.CreateService <IDefaultStepHost>();
// Setup container stephost and the right runtime variables for running job inside container.
if (ExecutionContext.Container != null)
{
if (handlerData is AgentPluginHandlerData)
{
// plugin handler always runs on the Host, the rumtime variables needs to the variable works on the Host, ex: file path variable System.DefaultWorkingDirectory
Dictionary <string, VariableValue> variableCopy = new Dictionary <string, VariableValue>(StringComparer.OrdinalIgnoreCase);
foreach (var publicVar in ExecutionContext.Variables.Public)
{
variableCopy[publicVar.Key] = new VariableValue(ExecutionContext.Container.TranslateToHostPath(publicVar.Value));
}
foreach (var secretVar in ExecutionContext.Variables.Private)
{
variableCopy[secretVar.Key] = new VariableValue(ExecutionContext.Container.TranslateToHostPath(secretVar.Value), true);
}
List <string> expansionWarnings;
runtimeVariables = new Variables(HostContext, variableCopy, out expansionWarnings);
expansionWarnings?.ForEach(x => ExecutionContext.Warning(x));
}
else if (handlerData is NodeHandlerData || handlerData is Node10HandlerData || handlerData is PowerShell3HandlerData)
{
// Only the node, node10, and powershell3 handlers support running inside container.
// Make sure required container is already created.
ArgUtil.NotNullOrEmpty(ExecutionContext.Container.ContainerId, nameof(ExecutionContext.Container.ContainerId));
var containerStepHost = HostContext.CreateService <IContainerStepHost>();
containerStepHost.Container = ExecutionContext.Container;
stepHost = containerStepHost;
}
else
{
throw new NotSupportedException(String.Format("Task '{0}' is using legacy execution handler '{1}' which is not supported in container execution flow.", definition.Data.FriendlyName, handlerData.GetType().ToString()));
}
}
// Load the default input values from the definition.
Trace.Verbose("Loading default inputs.");
var inputs = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var input in (definition.Data?.Inputs ?? new TaskInputDefinition[0]))
{
string key = input?.Name?.Trim() ?? string.Empty;
if (!string.IsNullOrEmpty(key))
{
inputs[key] = input.DefaultValue?.Trim() ?? string.Empty;
}
}
// Merge the instance inputs.
Trace.Verbose("Loading instance inputs.");
foreach (var input in (Task.Inputs as IEnumerable <KeyValuePair <string, string> > ?? new KeyValuePair <string, string> [0]))
{
string key = input.Key?.Trim() ?? string.Empty;
if (!string.IsNullOrEmpty(key))
{
inputs[key] = input.Value?.Trim() ?? string.Empty;
}
}
// Expand the inputs.
Trace.Verbose("Expanding inputs.");
runtimeVariables.ExpandValues(target: inputs);
VarUtil.ExpandEnvironmentVariables(HostContext, target: inputs);
// Translate the server file path inputs to local paths.
foreach (var input in definition.Data?.Inputs ?? new TaskInputDefinition[0])
{
if (string.Equals(input.InputType, TaskInputType.FilePath, StringComparison.OrdinalIgnoreCase))
{
Trace.Verbose($"Translating file path input '{input.Name}': '{inputs[input.Name]}'");
inputs[input.Name] = stepHost.ResolvePathForStepHost(TranslateFilePathInput(inputs[input.Name] ?? string.Empty));
Trace.Verbose($"Translated file path input '{input.Name}': '{inputs[input.Name]}'");
}
}
// Load the task environment.
Trace.Verbose("Loading task environment.");
var environment = new Dictionary <string, string>(VarUtil.EnvironmentVariableKeyComparer);
foreach (var env in (Task.Environment ?? new Dictionary <string, string>(0)))
{
string key = env.Key?.Trim() ?? string.Empty;
if (!string.IsNullOrEmpty(key))
{
environment[key] = env.Value?.Trim() ?? string.Empty;
}
}
// Expand the inputs.
Trace.Verbose("Expanding task environment.");
runtimeVariables.ExpandValues(target: environment);
VarUtil.ExpandEnvironmentVariables(HostContext, target: environment);
// Expand the handler inputs.
Trace.Verbose("Expanding handler inputs.");
VarUtil.ExpandValues(HostContext, source: inputs, target: handlerData.Inputs);
runtimeVariables.ExpandValues(target: handlerData.Inputs);
// Get each endpoint ID referenced by the task.
var endpointIds = new List <Guid>();
foreach (var input in definition.Data?.Inputs ?? new TaskInputDefinition[0])
{
if ((input.InputType ?? string.Empty).StartsWith("connectedService:", StringComparison.OrdinalIgnoreCase))
{
string inputKey = input?.Name?.Trim() ?? string.Empty;
string inputValue;
if (!string.IsNullOrEmpty(inputKey) &&
inputs.TryGetValue(inputKey, out inputValue) &&
!string.IsNullOrEmpty(inputValue))
{
foreach (string rawId in inputValue.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries))
{
Guid parsedId;
if (Guid.TryParse(rawId.Trim(), out parsedId) && parsedId != Guid.Empty)
{
endpointIds.Add(parsedId);
}
}
}
}
}
if (endpointIds.Count > 0 &&
(runtimeVariables.GetBoolean(WellKnownDistributedTaskVariables.RestrictSecrets) ?? false) &&
(runtimeVariables.GetBoolean(Microsoft.TeamFoundation.Build.WebApi.BuildVariables.IsFork) ?? false))
{
ExecutionContext.Result = TaskResult.Skipped;
ExecutionContext.ResultCode = $"References service endpoint. PRs from repository forks are not allowed to access secrets in the pipeline. For more information see https://go.microsoft.com/fwlink/?linkid=862029 ";
return;
}
// Get the endpoints referenced by the task.
var endpoints = (ExecutionContext.Endpoints ?? new List <ServiceEndpoint>(0))
.Join(inner: endpointIds,
outerKeySelector: (ServiceEndpoint endpoint) => endpoint.Id,
innerKeySelector: (Guid endpointId) => endpointId,
resultSelector: (ServiceEndpoint endpoint, Guid endpointId) => endpoint)
.ToList();
// Add the system endpoint.
foreach (ServiceEndpoint endpoint in (ExecutionContext.Endpoints ?? new List <ServiceEndpoint>(0)))
{
if (string.Equals(endpoint.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase))
{
endpoints.Add(endpoint);
break;
}
}
// Get each secure file ID referenced by the task.
var secureFileIds = new List <Guid>();
foreach (var input in definition.Data?.Inputs ?? new TaskInputDefinition[0])
{
if (string.Equals(input.InputType ?? string.Empty, "secureFile", StringComparison.OrdinalIgnoreCase))
{
string inputKey = input?.Name?.Trim() ?? string.Empty;
string inputValue;
if (!string.IsNullOrEmpty(inputKey) &&
inputs.TryGetValue(inputKey, out inputValue) &&
!string.IsNullOrEmpty(inputValue))
{
foreach (string rawId in inputValue.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries))
{
Guid parsedId;
if (Guid.TryParse(rawId.Trim(), out parsedId) && parsedId != Guid.Empty)
{
secureFileIds.Add(parsedId);
}
}
}
}
}
if (secureFileIds.Count > 0 &&
(runtimeVariables.GetBoolean(WellKnownDistributedTaskVariables.RestrictSecrets) ?? false) &&
(runtimeVariables.GetBoolean(Microsoft.TeamFoundation.Build.WebApi.BuildVariables.IsFork) ?? false))
{
ExecutionContext.Result = TaskResult.Skipped;
ExecutionContext.ResultCode = $"References secure file. PRs from repository forks are not allowed to access secrets in the pipeline. For more information see https://go.microsoft.com/fwlink/?linkid=862029";
return;
}
// Get the endpoints referenced by the task.
var secureFiles = (ExecutionContext.SecureFiles ?? new List <SecureFile>(0))
.Join(inner: secureFileIds,
outerKeySelector: (SecureFile secureFile) => secureFile.Id,
innerKeySelector: (Guid secureFileId) => secureFileId,
resultSelector: (SecureFile secureFile, Guid secureFileId) => secureFile)
.ToList();
// Set output variables.
foreach (var outputVar in definition.Data?.OutputVariables ?? new OutputVariable[0])
{
if (outputVar != null && !string.IsNullOrEmpty(outputVar.Name))
{
ExecutionContext.OutputVariables.Add(outputVar.Name);
}
}
// Create the handler.
IHandler handler = handlerFactory.Create(
ExecutionContext,
Task.Reference,
stepHost,
endpoints,
secureFiles,
handlerData,
inputs,
environment,
runtimeVariables,
taskDirectory: definition.Directory);
// Run the task.
await handler.RunAsync();
}
public async Task <TaskResult> RunAsync(JobRequestMessage message, CancellationToken jobRequestCancellationToken)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Environment, nameof(message.Environment));
ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables));
ArgUtil.NotNull(message.Tasks, nameof(message.Tasks));
Trace.Info("Job ID {0}", message.JobId);
if (message.Environment.Variables.ContainsKey(Constants.Variables.System.EnableAccessToken) &&
StringUtil.ConvertToBoolean(message.Environment.Variables[Constants.Variables.System.EnableAccessToken]))
{
// TODO: get access token use Util Method
message.Environment.Variables[Constants.Variables.System.AccessToken] = message.Environment.SystemConnection.Authorization.Parameters["AccessToken"];
}
// Setup the job server and job server queue.
var jobServer = HostContext.GetService <IJobServer>();
var jobServerCredential = ApiUtil.GetVssCredential(message.Environment.SystemConnection);
var jobConnection = ApiUtil.CreateConnection(ReplaceWithConfigUriBase(message.Environment.SystemConnection.Url), jobServerCredential);
await jobServer.ConnectAsync(jobConnection);
var jobServerQueue = HostContext.GetService <IJobServerQueue>();
jobServerQueue.Start(message);
IExecutionContext jobContext = null;
try
{
// Create the job execution context.
jobContext = HostContext.CreateService <IExecutionContext>();
jobContext.InitializeJob(message, jobRequestCancellationToken);
Trace.Info("Starting the job execution context.");
jobContext.Start();
// Set agent variables.
AgentSettings settings = HostContext.GetService <IConfigurationStore>().GetSettings();
jobContext.Variables.Set(Constants.Variables.Agent.Id, settings.AgentId.ToString(CultureInfo.InvariantCulture));
jobContext.Variables.Set(Constants.Variables.Agent.HomeDirectory, IOUtil.GetRootPath());
jobContext.Variables.Set(Constants.Variables.Agent.JobName, message.JobName);
jobContext.Variables.Set(Constants.Variables.Agent.MachineName, Environment.MachineName);
jobContext.Variables.Set(Constants.Variables.Agent.Name, settings.AgentName);
jobContext.Variables.Set(Constants.Variables.Agent.RootDirectory, IOUtil.GetWorkPath(HostContext));
jobContext.Variables.Set(Constants.Variables.Agent.ServerOMDirectory, Path.Combine(IOUtil.GetExternalsPath(), "vstsom"));
jobContext.Variables.Set(Constants.Variables.Agent.WorkFolder, IOUtil.GetWorkPath(HostContext));
jobContext.Variables.Set(Constants.Variables.System.WorkFolder, IOUtil.GetWorkPath(HostContext));
// prefer task definitions url, then TFS url
var taskServer = HostContext.GetService <ITaskServer>();
string taskUrl = jobContext.Variables.System_TaskDefinitionsUri;
Uri taskServerUri;
if (string.IsNullOrEmpty(taskUrl))
{
taskServerUri = ReplaceWithConfigUriBase(message.Environment.SystemConnection.Url);
Trace.Info($"Creating task server with tfs server url {taskServerUri.ToString()}");
}
else
{
taskServerUri = ReplaceWithConfigUriBase(new Uri(taskUrl));
Trace.Info($"Creating task server with {taskServerUri.ToString()}");
}
var taskServerCredential = ApiUtil.GetVssCredential(message.Environment.SystemConnection);
await taskServer.ConnectAsync(ApiUtil.CreateConnection(taskServerUri, taskServerCredential));
// Expand the endpoint data values.
foreach (ServiceEndpoint endpoint in jobContext.Endpoints)
{
jobContext.Variables.ExpandValues(target: endpoint.Data);
VarUtil.ExpandEnvironmentVariables(HostContext, target: endpoint.Data);
}
// Get the job extensions.
Trace.Info("Getting job extensions.");
string hostType = jobContext.Variables.System_HostType;
var extensionManager = HostContext.GetService <IExtensionManager>();
IJobExtension[] extensions =
(extensionManager.GetExtensions <IJobExtension>() ?? new List <IJobExtension>())
.Where(x => string.Equals(x.HostType, hostType, StringComparison.OrdinalIgnoreCase))
.ToArray();
// Add the prepare steps.
Trace.Info("Adding job prepare extensions.");
List <IStep> steps = new List <IStep>();
foreach (IJobExtension extension in extensions)
{
if (extension.PrepareStep != null)
{
Trace.Verbose($"Adding {extension.GetType().Name}.{nameof(extension.PrepareStep)}.");
extension.PrepareStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), extension.PrepareStep.DisplayName);
steps.Add(extension.PrepareStep);
}
}
// Add the task steps.
Trace.Info("Adding tasks.");
foreach (TaskInstance taskInstance in message.Tasks)
{
Trace.Verbose($"Adding {taskInstance.DisplayName}.");
var taskRunner = HostContext.CreateService <ITaskRunner>();
taskRunner.ExecutionContext = jobContext.CreateChild(taskInstance.InstanceId, taskInstance.DisplayName);
taskRunner.TaskInstance = taskInstance;
steps.Add(taskRunner);
}
// Add the finally steps.
Trace.Info("Adding job finally extensions.");
foreach (IJobExtension extension in extensions)
{
if (extension.FinallyStep != null)
{
Trace.Verbose($"Adding {extension.GetType().Name}.{nameof(extension.FinallyStep)}.");
extension.FinallyStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), extension.FinallyStep.DisplayName);
steps.Add(extension.FinallyStep);
}
}
// Download tasks if not already in the cache
Trace.Info("Downloading task definitions.");
var taskManager = HostContext.GetService <ITaskManager>();
try
{
await taskManager.DownloadAsync(jobContext, message.Tasks);
}
catch (OperationCanceledException ex)
{
// set the job to canceled
Trace.Error($"Caught exception: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Canceled));
}
catch (Exception ex)
{
// Log the error and fail the job.
Trace.Error($"Caught exception from {nameof(TaskManager)}: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Failed));
}
// Run the steps.
var stepsRunner = HostContext.GetService <IStepsRunner>();
try
{
await stepsRunner.RunAsync(jobContext, steps);
}
catch (OperationCanceledException ex)
{
// set the job to canceled
Trace.Error($"Caught exception: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Canceled));
}
catch (Exception ex)
{
// Log the error and fail the job.
Trace.Error($"Caught exception from {nameof(StepsRunner)}: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Failed));
}
Trace.Info($"Job result: {jobContext.Result}");
// Complete the job.
Trace.Info("Completing the job execution context.");
return(jobContext.Complete());
}
finally
{
// Drain the job server queue.
if (jobServerQueue != null)
{
try
{
Trace.Info("Shutting down the job server queue.");
await jobServerQueue.ShutdownAsync();
}
catch (Exception ex)
{
Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(jobServerQueue.ShutdownAsync)}: {ex}");
}
}
}
}
