public async Task RunMaintenanceOperation(IExecutionContext executionContext)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
// this might be not accurate when the agent is configured for old TFS server
int totalAvailableTimeInMinutes = executionContext.Variables.GetInt("maintenance.jobtimeoutinminutes") ?? 60;
// start a timer to track how much time we used
Stopwatch totalTimeSpent = Stopwatch.StartNew();
var trackingManager = HostContext.GetService <ITrackingManager>();
int staleBuildDirThreshold = executionContext.Variables.GetInt("maintenance.deleteworkingdirectory.daysthreshold") ?? 0;
if (staleBuildDirThreshold > 0)
{
// scan unused build directories
executionContext.Output(StringUtil.Loc("DiscoverBuildDir", staleBuildDirThreshold));
trackingManager.MarkExpiredForGarbageCollection(executionContext, TimeSpan.FromDays(staleBuildDirThreshold));
}
else
{
executionContext.Output(StringUtil.Loc("GCBuildDirNotEnabled"));
return;
}
executionContext.Output(StringUtil.Loc("GCBuildDir"));
// delete unused build directories
trackingManager.DisposeCollectedGarbage(executionContext);
// give source provider a chance to run maintenance operation
Trace.Info("Scan all SourceFolder tracking files.");
string searchRoot = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), Constants.Build.Path.SourceRootMappingDirectory);
if (!Directory.Exists(searchRoot))
{
executionContext.Output(StringUtil.Loc("GCDirNotExist", searchRoot));
return;
}
// <tracking config, tracking file path>
List <Tuple <TrackingConfig, string> > optimizeTrackingFiles = new List <Tuple <TrackingConfig, string> >();
var allTrackingFiles = Directory.EnumerateFiles(searchRoot, Constants.Build.Path.TrackingConfigFile, SearchOption.AllDirectories);
Trace.Verbose($"Find {allTrackingFiles.Count()} tracking files.");
foreach (var trackingFile in allTrackingFiles)
{
executionContext.Output(StringUtil.Loc("EvaluateTrackingFile", trackingFile));
TrackingConfigBase tracking = trackingManager.LoadIfExists(executionContext, trackingFile);
// detect whether the tracking file is in new format.
TrackingConfig newTracking = tracking as TrackingConfig;
if (newTracking == null)
{
executionContext.Output(StringUtil.Loc("GCOldFormatTrackingFile", trackingFile));
}
else if (string.IsNullOrEmpty(newTracking.RepositoryType))
{
// repository not been set.
executionContext.Output(StringUtil.Loc("SkipTrackingFileWithoutRepoType", trackingFile));
}
else
{
optimizeTrackingFiles.Add(new Tuple <TrackingConfig, string>(newTracking, trackingFile));
}
}
// Sort the all tracking file ASC by last maintenance attempted time
foreach (var trackingInfo in optimizeTrackingFiles.OrderBy(x => x.Item1.LastMaintenanceAttemptedOn))
{
// maintenance has been cancelled.
executionContext.CancellationToken.ThrowIfCancellationRequested();
bool runMainenance = false;
TrackingConfig trackingConfig = trackingInfo.Item1;
string trackingFile = trackingInfo.Item2;
if (trackingConfig.LastMaintenanceAttemptedOn == null)
{
// this folder never run maintenance before, we will do maintenance if there is more than half of the time remains.
if (totalTimeSpent.Elapsed.TotalMinutes < totalAvailableTimeInMinutes / 2) // 50% time left
{
runMainenance = true;
}
else
{
executionContext.Output($"Working directory '{trackingConfig.BuildDirectory}' has never run maintenance before. Skip since we may not have enough time.");
}
}
else if (trackingConfig.LastMaintenanceCompletedOn == null)
{
// this folder did finish maintenance last time, this might indicate we need more time for this working directory
if (totalTimeSpent.Elapsed.TotalMinutes < totalAvailableTimeInMinutes / 4) // 75% time left
{
runMainenance = true;
}
else
{
executionContext.Output($"Working directory '{trackingConfig.BuildDirectory}' didn't finish maintenance last time. Skip since we may not have enough time.");
}
}
else
{
// estimate time for running maintenance
TimeSpan estimateTime = trackingConfig.LastMaintenanceCompletedOn.Value - trackingConfig.LastMaintenanceAttemptedOn.Value;
// there is more than 10 mins left after we run maintenance on this repository directory
if (totalAvailableTimeInMinutes > totalTimeSpent.Elapsed.TotalMinutes + estimateTime.TotalMinutes + 10)
{
runMainenance = true;
}
else
{
executionContext.Output($"Working directory '{trackingConfig.BuildDirectory}' may take about '{estimateTime.TotalMinutes}' mins to finish maintenance. It's too risky since we only have '{totalAvailableTimeInMinutes - totalTimeSpent.Elapsed.TotalMinutes}' mins left for maintenance.");
}
}
if (runMainenance)
{
var extensionManager = HostContext.GetService <IExtensionManager>();
ISourceProvider sourceProvider = extensionManager.GetExtensions <ISourceProvider>().FirstOrDefault(x => string.Equals(x.RepositoryType, trackingConfig.RepositoryType, StringComparison.OrdinalIgnoreCase));
if (sourceProvider != null)
{
try
{
trackingManager.MaintenanceStarted(trackingConfig, trackingFile);
string repositoryPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), trackingConfig.SourcesDirectory);
await sourceProvider.RunMaintenanceOperations(executionContext, repositoryPath);
trackingManager.MaintenanceCompleted(trackingConfig, trackingFile);
}
catch (Exception ex)
{
executionContext.Error(StringUtil.Loc("ErrorDuringBuildGC", trackingFile));
executionContext.Error(ex);
}
}
}
}
}
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_jobServerQueue = HostContext.GetService <IJobServerQueue>();
}
public async Task <TaskResult> RunAsync(AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Environment, nameof(message.Environment));
ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables));
ArgUtil.NotNull(message.Tasks, nameof(message.Tasks));
Trace.Info("Job ID {0}", message.JobId);
if (message.Environment.Variables.ContainsKey(Constants.Variables.System.EnableAccessToken) &&
StringUtil.ConvertToBoolean(message.Environment.Variables[Constants.Variables.System.EnableAccessToken]))
{
// TODO: get access token use Util Method
message.Environment.Variables[Constants.Variables.System.AccessToken] = message.Environment.SystemConnection.Authorization.Parameters["AccessToken"];
}
// Make sure SystemConnection Url and Endpoint Url match Config Url base
ReplaceConfigUriBaseInJobRequestMessage(message);
// Setup the job server and job server queue.
var jobServer = HostContext.GetService <IJobServer>();
var jobServerCredential = ApiUtil.GetVssCredential(message.Environment.SystemConnection);
Uri jobServerUrl = message.Environment.SystemConnection.Url;
Trace.Info($"Creating job server with URL: {jobServerUrl}");
var jobConnection = ApiUtil.CreateConnection(jobServerUrl, jobServerCredential);
await jobServer.ConnectAsync(jobConnection);
var jobServerQueue = HostContext.GetService <IJobServerQueue>();
jobServerQueue.Start(message);
IExecutionContext jobContext = null;
try
{
// Create the job execution context.
jobContext = HostContext.CreateService <IExecutionContext>();
jobContext.InitializeJob(message, jobRequestCancellationToken);
Trace.Info("Starting the job execution context.");
jobContext.Start();
// Print agent version into log for better diagnostic experience
jobContext.Output(StringUtil.Loc("AgentVersion", Constants.Agent.Version));
// Set agent variables.
AgentSettings settings = HostContext.GetService <IConfigurationStore>().GetSettings();
jobContext.Variables.Set(Constants.Variables.Agent.Id, settings.AgentId.ToString(CultureInfo.InvariantCulture));
jobContext.Variables.Set(Constants.Variables.Agent.HomeDirectory, IOUtil.GetRootPath());
jobContext.Variables.Set(Constants.Variables.Agent.JobName, message.JobName);
jobContext.Variables.Set(Constants.Variables.Agent.MachineName, Environment.MachineName);
jobContext.Variables.Set(Constants.Variables.Agent.Name, settings.AgentName);
jobContext.Variables.Set(Constants.Variables.Agent.RootDirectory, IOUtil.GetWorkPath(HostContext));
#if OS_WINDOWS
jobContext.Variables.Set(Constants.Variables.Agent.ServerOMDirectory, Path.Combine(IOUtil.GetExternalsPath(), Constants.Path.ServerOMDirectory));
#endif
jobContext.Variables.Set(Constants.Variables.Agent.WorkFolder, IOUtil.GetWorkPath(HostContext));
jobContext.Variables.Set(Constants.Variables.System.WorkFolder, IOUtil.GetWorkPath(HostContext));
// prefer task definitions url, then TFS collection url, then TFS account url
var taskServer = HostContext.GetService <ITaskServer>();
Uri taskServerUri = null;
if (!string.IsNullOrEmpty(jobContext.Variables.System_TaskDefinitionsUri))
{
taskServerUri = new Uri(jobContext.Variables.System_TaskDefinitionsUri);
}
else if (!string.IsNullOrEmpty(jobContext.Variables.System_TFCollectionUrl))
{
taskServerUri = new Uri(jobContext.Variables.System_TFCollectionUrl);
}
var taskServerCredential = ApiUtil.GetVssCredential(message.Environment.SystemConnection);
if (taskServerUri != null)
{
Trace.Info($"Creating task server with {taskServerUri}");
await taskServer.ConnectAsync(ApiUtil.CreateConnection(taskServerUri, taskServerCredential));
}
if (taskServerUri == null || !await taskServer.TaskDefinitionEndpointExist(jobRequestCancellationToken))
{
Trace.Info($"Can't determine task download url from JobMessage or the endpoint doesn't exist.");
var configStore = HostContext.GetService <IConfigurationStore>();
taskServerUri = new Uri(configStore.GetSettings().ServerUrl);
Trace.Info($"Recreate task server with configuration server url: {taskServerUri}");
await taskServer.ConnectAsync(ApiUtil.CreateConnection(taskServerUri, taskServerCredential));
}
// Expand the endpoint data values.
foreach (ServiceEndpoint endpoint in jobContext.Endpoints)
{
jobContext.Variables.ExpandValues(target: endpoint.Data);
VarUtil.ExpandEnvironmentVariables(HostContext, target: endpoint.Data);
}
// Get the job extensions.
Trace.Info("Getting job extensions.");
string hostType = jobContext.Variables.System_HostType;
var extensionManager = HostContext.GetService <IExtensionManager>();
IJobExtension[] extensions =
(extensionManager.GetExtensions <IJobExtension>() ?? new List <IJobExtension>())
.Where(x => string.Equals(x.HostType, hostType, StringComparison.OrdinalIgnoreCase))
.ToArray();
// Add the prepare steps.
Trace.Info("Adding job prepare extensions.");
List <IStep> steps = new List <IStep>();
foreach (IJobExtension extension in extensions)
{
if (extension.PrepareStep != null)
{
Trace.Verbose($"Adding {extension.GetType().Name}.{nameof(extension.PrepareStep)}.");
extension.PrepareStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), extension.PrepareStep.DisplayName);
steps.Add(extension.PrepareStep);
}
}
// Add the task steps.
Trace.Info("Adding tasks.");
foreach (TaskInstance taskInstance in message.Tasks)
{
Trace.Verbose($"Adding {taskInstance.DisplayName}.");
var taskRunner = HostContext.CreateService <ITaskRunner>();
taskRunner.ExecutionContext = jobContext.CreateChild(taskInstance.InstanceId, taskInstance.DisplayName);
taskRunner.TaskInstance = taskInstance;
steps.Add(taskRunner);
}
// Add the finally steps.
Trace.Info("Adding job finally extensions.");
foreach (IJobExtension extension in extensions)
{
if (extension.FinallyStep != null)
{
Trace.Verbose($"Adding {extension.GetType().Name}.{nameof(extension.FinallyStep)}.");
extension.FinallyStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), extension.FinallyStep.DisplayName);
steps.Add(extension.FinallyStep);
}
}
// Download tasks if not already in the cache
Trace.Info("Downloading task definitions.");
var taskManager = HostContext.GetService <ITaskManager>();
try
{
await taskManager.DownloadAsync(jobContext, message.Tasks);
}
catch (OperationCanceledException ex)
{
// set the job to canceled
Trace.Error($"Caught exception: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Canceled));
}
catch (Exception ex)
{
// Log the error and fail the job.
Trace.Error($"Caught exception from {nameof(TaskManager)}: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Failed));
}
// Run the steps.
var stepsRunner = HostContext.GetService <IStepsRunner>();
try
{
await stepsRunner.RunAsync(jobContext, steps);
}
catch (OperationCanceledException ex)
{
// set the job to canceled
Trace.Error($"Caught exception: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Canceled));
}
catch (Exception ex)
{
// Log the error and fail the job.
Trace.Error($"Caught exception from {nameof(StepsRunner)}: {ex}");
jobContext.Error(ex);
return(jobContext.Complete(TaskResult.Failed));
}
Trace.Info($"Job result: {jobContext.Result}");
// Complete the job.
Trace.Info("Completing the job execution context.");
return(jobContext.Complete());
}
finally
{
// Drain the job server queue.
if (jobServerQueue != null)
{
try
{
Trace.Info("Shutting down the job server queue.");
await jobServerQueue.ShutdownAsync();
}
catch (Exception ex)
{
Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(jobServerQueue.ShutdownAsync)}: {ex}");
}
}
}
}
private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, Pipelines.ActionStep repositoryAction)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
var repositoryReference = repositoryAction.Reference as Pipelines.RepositoryPathReference;
ArgUtil.NotNull(repositoryReference, nameof(repositoryReference));
if (string.Equals(repositoryReference.RepositoryType, Pipelines.PipelineConstants.SelfAlias, StringComparison.OrdinalIgnoreCase))
{
Trace.Info($"Repository action is in 'self' repository.");
return;
}
if (!string.Equals(repositoryReference.RepositoryType, Pipelines.RepositoryTypes.GitHub, StringComparison.OrdinalIgnoreCase))
{
throw new NotSupportedException(repositoryReference.RepositoryType);
}
ArgUtil.NotNullOrEmpty(repositoryReference.Name, nameof(repositoryReference.Name));
ArgUtil.NotNullOrEmpty(repositoryReference.Ref, nameof(repositoryReference.Ref));
string destDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), repositoryReference.Name.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), repositoryReference.Ref);
string watermarkFile = GetWatermarkFilePath(destDirectory);
if (File.Exists(watermarkFile))
{
executionContext.Debug($"Action '{repositoryReference.Name}@{repositoryReference.Ref}' already downloaded at '{destDirectory}'.");
return;
}
else
{
// make sure we get a clean folder ready to use.
IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken);
Directory.CreateDirectory(destDirectory);
executionContext.Output($"Download action repository '{repositoryReference.Name}@{repositoryReference.Ref}'");
}
var configurationStore = HostContext.GetService <IConfigurationStore>();
var isHostedServer = configurationStore.GetSettings().IsHostedServer;
if (isHostedServer)
{
string apiUrl = GetApiUrl(executionContext);
string archiveLink = BuildLinkToActionArchive(apiUrl, repositoryReference.Name, repositoryReference.Ref);
Trace.Info($"Download archive '{archiveLink}' to '{destDirectory}'.");
await DownloadRepositoryActionAsync(executionContext, archiveLink, destDirectory);
return;
}
else
{
string apiUrl = GetApiUrl(executionContext);
// URLs to try:
var archiveLinks = new List <string> {
// A built-in action or an action the user has created, on their GHES instance
// Example: https://my-ghes/api/v3/repos/my-org/my-action/tarball/v1
BuildLinkToActionArchive(apiUrl, repositoryReference.Name, repositoryReference.Ref),
// A community action, synced to their GHES instance
// Example: https://my-ghes/api/v3/repos/actions-community/some-org-some-action/tarball/v1
BuildLinkToActionArchive(apiUrl, $"actions-community/{repositoryReference.Name.Replace("/", "-")}", repositoryReference.Ref)
};
foreach (var archiveLink in archiveLinks)
{
Trace.Info($"Download archive '{archiveLink}' to '{destDirectory}'.");
try
{
await DownloadRepositoryActionAsync(executionContext, archiveLink, destDirectory);
return;
}
catch (ActionNotFoundException)
{
Trace.Info($"Failed to find the action '{repositoryReference.Name}' at ref '{repositoryReference.Ref}' at {archiveLink}");
continue;
}
}
throw new ActionNotFoundException($"Failed to find the action '{repositoryReference.Name}' at ref '{repositoryReference.Ref}'. Paths attempted: {string.Join(", ", archiveLinks)}");
}
}
public async Task <List <JobExtensionRunner> > PrepareActionsAsync(IExecutionContext executionContext, IEnumerable <Pipelines.JobStep> steps)
{
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(steps, nameof(steps));
executionContext.Output("Prepare all required actions");
Dictionary <string, List <Guid> > imagesToPull = new Dictionary <string, List <Guid> >(StringComparer.OrdinalIgnoreCase);
Dictionary <string, List <Guid> > imagesToBuild = new Dictionary <string, List <Guid> >(StringComparer.OrdinalIgnoreCase);
Dictionary <string, ActionContainer> imagesToBuildInfo = new Dictionary <string, ActionContainer>(StringComparer.OrdinalIgnoreCase);
List <JobExtensionRunner> containerSetupSteps = new List <JobExtensionRunner>();
IEnumerable <Pipelines.ActionStep> actions = steps.OfType <Pipelines.ActionStep>();
// TODO: Deprecate the PREVIEW_ACTION_TOKEN
// Log even if we aren't using it to ensure users know.
if (!string.IsNullOrEmpty(executionContext.Variables.Get("PREVIEW_ACTION_TOKEN")))
{
executionContext.Warning("The 'PREVIEW_ACTION_TOKEN' secret is deprecated. Please remove it from the repository's secrets");
}
// Clear the cache (for self-hosted runners)
// Note, temporarily avoid this step for the on-premises product, to avoid rate limiting.
var configurationStore = HostContext.GetService <IConfigurationStore>();
var isHostedServer = configurationStore.GetSettings().IsHostedServer;
if (isHostedServer)
{
IOUtil.DeleteDirectory(HostContext.GetDirectory(WellKnownDirectory.Actions), executionContext.CancellationToken);
}
foreach (var action in actions)
{
if (action.Reference.Type == Pipelines.ActionSourceType.ContainerRegistry)
{
ArgUtil.NotNull(action, nameof(action));
var containerReference = action.Reference as Pipelines.ContainerRegistryReference;
ArgUtil.NotNull(containerReference, nameof(containerReference));
ArgUtil.NotNullOrEmpty(containerReference.Image, nameof(containerReference.Image));
if (!imagesToPull.ContainsKey(containerReference.Image))
{
imagesToPull[containerReference.Image] = new List <Guid>();
}
Trace.Info($"Action {action.Name} ({action.Id}) needs to pull image '{containerReference.Image}'");
imagesToPull[containerReference.Image].Add(action.Id);
}
else if (action.Reference.Type == Pipelines.ActionSourceType.Repository)
{
// only download the repository archive
await DownloadRepositoryActionAsync(executionContext, action);
// more preparation base on content in the repository (action.yml)
var setupInfo = PrepareRepositoryActionAsync(executionContext, action);
if (setupInfo != null)
{
if (!string.IsNullOrEmpty(setupInfo.Image))
{
if (!imagesToPull.ContainsKey(setupInfo.Image))
{
imagesToPull[setupInfo.Image] = new List <Guid>();
}
Trace.Info($"Action {action.Name} ({action.Id}) from repository '{setupInfo.ActionRepository}' needs to pull image '{setupInfo.Image}'");
imagesToPull[setupInfo.Image].Add(action.Id);
}
else
{
ArgUtil.NotNullOrEmpty(setupInfo.ActionRepository, nameof(setupInfo.ActionRepository));
if (!imagesToBuild.ContainsKey(setupInfo.ActionRepository))
{
imagesToBuild[setupInfo.ActionRepository] = new List <Guid>();
}
Trace.Info($"Action {action.Name} ({action.Id}) from repository '{setupInfo.ActionRepository}' needs to build image '{setupInfo.Dockerfile}'");
imagesToBuild[setupInfo.ActionRepository].Add(action.Id);
imagesToBuildInfo[setupInfo.ActionRepository] = setupInfo;
}
}
}
}
if (imagesToPull.Count > 0)
{
foreach (var imageToPull in imagesToPull)
{
Trace.Info($"{imageToPull.Value.Count} steps need to pull image '{imageToPull.Key}'");
containerSetupSteps.Add(new JobExtensionRunner(runAsync: this.PullActionContainerAsync,
condition: $"{PipelineTemplateConstants.Success}()",
displayName: $"Pull {imageToPull.Key}",
data: new ContainerSetupInfo(imageToPull.Value, imageToPull.Key)));
}
}
if (imagesToBuild.Count > 0)
{
foreach (var imageToBuild in imagesToBuild)
{
var setupInfo = imagesToBuildInfo[imageToBuild.Key];
Trace.Info($"{imageToBuild.Value.Count} steps need to build image from '{setupInfo.Dockerfile}'");
containerSetupSteps.Add(new JobExtensionRunner(runAsync: this.BuildActionContainerAsync,
condition: $"{PipelineTemplateConstants.Success}()",
displayName: $"Build {setupInfo.ActionRepository}",
data: new ContainerSetupInfo(imageToBuild.Value, setupInfo.Dockerfile, setupInfo.WorkingDirectory)));
}
}
#if !OS_LINUX
if (containerSetupSteps.Count > 0)
{
executionContext.Output("Container action is only supported on Linux, skip pull and build docker images.");
containerSetupSteps.Clear();
}
#endif
return(containerSetupSteps);
}
public async Task RunAsync()
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
ArgUtil.NotNull(ExecutionContext.Variables, nameof(ExecutionContext.Variables));
ArgUtil.NotNull(TaskInstance, nameof(TaskInstance));
var taskManager = HostContext.GetService <ITaskManager>();
var handlerFactory = HostContext.GetService <IHandlerFactory>();
// Set the task display name variable.
ExecutionContext.Variables.Set(Constants.Variables.Task.DisplayName, DisplayName);
// Load the task definition and choose the handler.
// TODO: Add a try catch here to give a better error message.
Definition definition = taskManager.Load(TaskInstance);
ArgUtil.NotNull(definition, nameof(definition));
HandlerData handlerData =
definition.Data?.Execution?.All
.OrderBy(x => !x.PreferredOnCurrentPlatform()) // Sort true to false.
.ThenBy(x => x.Priority)
.FirstOrDefault();
if (handlerData == null)
{
throw new Exception(StringUtil.Loc("SupportedTaskHandlerNotFound"));
}
// Load the default input values from the definition.
Trace.Verbose("Loading default inputs.");
var inputs = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var input in (definition.Data?.Inputs ?? new TaskInputDefinition[0]))
{
string key = input?.Name?.Trim() ?? string.Empty;
if (!string.IsNullOrEmpty(key))
{
inputs[key] = input.DefaultValue?.Trim() ?? string.Empty;
}
}
// Merge the instance inputs.
Trace.Verbose("Loading instance inputs.");
foreach (var input in (TaskInstance.Inputs as IEnumerable <KeyValuePair <string, string> > ?? new KeyValuePair <string, string> [0]))
{
string key = input.Key?.Trim() ?? string.Empty;
if (!string.IsNullOrEmpty(key))
{
inputs[key] = input.Value?.Trim() ?? string.Empty;
}
}
// Expand the inputs.
Trace.Verbose("Expanding inputs.");
ExecutionContext.Variables.ExpandValues(target: inputs);
VarUtil.ExpandEnvironmentVariables(HostContext, target: inputs);
// Translate the server file path inputs to local paths.
foreach (var input in definition.Data?.Inputs ?? new TaskInputDefinition[0])
{
if (string.Equals(input.InputType, TaskInputType.FilePath, StringComparison.OrdinalIgnoreCase))
{
Trace.Verbose($"Translating file path input '{input.Name}': '{inputs[input.Name]}'");
inputs[input.Name] = TranslateFilePathInput(inputs[input.Name] ?? string.Empty);
Trace.Verbose($"Translated file path input '{input.Name}': '{inputs[input.Name]}'");
}
}
// Expand the handler inputs.
Trace.Verbose("Expanding handler inputs.");
VarUtil.ExpandValues(HostContext, source: inputs, target: handlerData.Inputs);
// Create the handler.
IHandler handler = handlerFactory.Create(
ExecutionContext,
handlerData,
inputs,
taskDirectory: definition.Directory,
filePathInputRootDirectory: TranslateFilePathInput(string.Empty));
// Run the task.
await handler.RunAsync();
}
public async Task ConfigureAsync(CommandSettings command)
{
ArgUtil.Equal(RunMode.Normal, HostContext.RunMode, nameof(HostContext.RunMode));
Trace.Info(nameof(ConfigureAsync));
if (IsConfigured())
{
throw new InvalidOperationException(StringUtil.Loc("AlreadyConfiguredError"));
}
// Populate proxy setting from commandline args
var vstsProxy = HostContext.GetService <IVstsAgentWebProxy>();
bool saveProxySetting = false;
string proxyUrl = command.GetProxyUrl();
if (!string.IsNullOrEmpty(proxyUrl))
{
if (!Uri.IsWellFormedUriString(proxyUrl, UriKind.Absolute))
{
throw new ArgumentOutOfRangeException(nameof(proxyUrl));
}
Trace.Info("Reset proxy base on commandline args.");
string proxyUserName = command.GetProxyUserName();
string proxyPassword = command.GetProxyPassword();
(vstsProxy as VstsAgentWebProxy).SetupProxy(proxyUrl, proxyUserName, proxyPassword);
saveProxySetting = true;
}
// Populate cert setting from commandline args
var agentCertManager = HostContext.GetService <IAgentCertificateManager>();
bool saveCertSetting = false;
bool skipCertValidation = command.GetSkipCertificateValidation();
string caCert = command.GetCACertificate();
string clientCert = command.GetClientCertificate();
string clientCertKey = command.GetClientCertificatePrivateKey();
string clientCertArchive = command.GetClientCertificateArchrive();
string clientCertPassword = command.GetClientCertificatePassword();
// We require all Certificate files are under agent root.
// So we can set ACL correctly when configure as service
if (!string.IsNullOrEmpty(caCert))
{
caCert = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), caCert);
ArgUtil.File(caCert, nameof(caCert));
}
if (!string.IsNullOrEmpty(clientCert) &&
!string.IsNullOrEmpty(clientCertKey) &&
!string.IsNullOrEmpty(clientCertArchive))
{
// Ensure all client cert pieces are there.
clientCert = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), clientCert);
clientCertKey = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), clientCertKey);
clientCertArchive = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), clientCertArchive);
ArgUtil.File(clientCert, nameof(clientCert));
ArgUtil.File(clientCertKey, nameof(clientCertKey));
ArgUtil.File(clientCertArchive, nameof(clientCertArchive));
}
else if (!string.IsNullOrEmpty(clientCert) ||
!string.IsNullOrEmpty(clientCertKey) ||
!string.IsNullOrEmpty(clientCertArchive))
{
// Print out which args are missing.
ArgUtil.NotNullOrEmpty(Constants.Agent.CommandLine.Args.SslClientCert, Constants.Agent.CommandLine.Args.SslClientCert);
ArgUtil.NotNullOrEmpty(Constants.Agent.CommandLine.Args.SslClientCertKey, Constants.Agent.CommandLine.Args.SslClientCertKey);
ArgUtil.NotNullOrEmpty(Constants.Agent.CommandLine.Args.SslClientCertArchive, Constants.Agent.CommandLine.Args.SslClientCertArchive);
}
if (skipCertValidation || !string.IsNullOrEmpty(caCert) || !string.IsNullOrEmpty(clientCert))
{
Trace.Info("Reset agent cert setting base on commandline args.");
(agentCertManager as AgentCertificateManager).SetupCertificate(skipCertValidation, caCert, clientCert, clientCertKey, clientCertArchive, clientCertPassword);
saveCertSetting = true;
}
AgentSettings agentSettings = new AgentSettings();
// TEE EULA
agentSettings.AcceptTeeEula = false;
switch (Constants.Agent.Platform)
{
case Constants.OSPlatform.OSX:
case Constants.OSPlatform.Linux:
// Write the section header.
WriteSection(StringUtil.Loc("EulasSectionHeader"));
// Verify the EULA exists on disk in the expected location.
string eulaFile = Path.Combine(IOUtil.GetExternalsPath(), Constants.Path.TeeDirectory, "license.html");
ArgUtil.File(eulaFile, nameof(eulaFile));
// Write elaborate verbiage about the TEE EULA.
_term.WriteLine(StringUtil.Loc("TeeEula", eulaFile));
_term.WriteLine();
// Prompt to acccept the TEE EULA.
agentSettings.AcceptTeeEula = command.GetAcceptTeeEula();
break;
case Constants.OSPlatform.Windows:
// Warn and continue if .NET 4.6 is not installed.
var netFrameworkUtil = HostContext.GetService <INetFrameworkUtil>();
if (!netFrameworkUtil.Test(new Version(4, 6)))
{
WriteSection(StringUtil.Loc("PrerequisitesSectionHeader")); // Section header.
_term.WriteLine(StringUtil.Loc("MinimumNetFrameworkTfvc")); // Warning.
}
break;
default:
throw new NotSupportedException();
}
// Create the configuration provider as per agent type.
string agentType;
if (command.DeploymentGroup)
{
agentType = Constants.Agent.AgentConfigurationProvider.DeploymentAgentConfiguration;
}
else if (command.DeploymentPool)
{
agentType = Constants.Agent.AgentConfigurationProvider.SharedDeploymentAgentConfiguration;
}
else
{
agentType = Constants.Agent.AgentConfigurationProvider.BuildReleasesAgentConfiguration;
}
var extensionManager = HostContext.GetService <IExtensionManager>();
IConfigurationProvider agentProvider =
(extensionManager.GetExtensions <IConfigurationProvider>())
.FirstOrDefault(x => x.ConfigurationProviderType == agentType);
ArgUtil.NotNull(agentProvider, agentType);
// Loop getting url and creds until you can connect
ICredentialProvider credProvider = null;
VssCredentials creds = null;
WriteSection(StringUtil.Loc("ConnectSectionHeader"));
while (true)
{
// Get the URL
agentProvider.GetServerUrl(agentSettings, command);
// Get the credentials
credProvider = GetCredentialProvider(command, agentSettings.ServerUrl);
creds = credProvider.GetVssCredentials(HostContext);
Trace.Info("cred retrieved");
try
{
// Validate can connect.
await agentProvider.TestConnectionAsync(agentSettings, creds);
Trace.Info("Test Connection complete.");
break;
}
catch (Exception e) when(!command.Unattended)
{
_term.WriteError(e);
_term.WriteError(StringUtil.Loc("FailedToConnect"));
}
}
_agentServer = HostContext.GetService <IAgentServer>();
// We want to use the native CSP of the platform for storage, so we use the RSACSP directly
RSAParameters publicKey;
var keyManager = HostContext.GetService <IRSAKeyManager>();
using (var rsa = keyManager.CreateKey())
{
publicKey = rsa.ExportParameters(false);
}
// Loop getting agent name and pool name
WriteSection(StringUtil.Loc("RegisterAgentSectionHeader"));
while (true)
{
try
{
await agentProvider.GetPoolId(agentSettings, command);
break;
}
catch (Exception e) when(!command.Unattended)
{
_term.WriteError(e);
_term.WriteError(agentProvider.GetFailedToFindPoolErrorString());
}
}
TaskAgent agent;
while (true)
{
agentSettings.AgentName = command.GetAgentName();
// Get the system capabilities.
// TODO: Hook up to ctrl+c cancellation token.
_term.WriteLine(StringUtil.Loc("ScanToolCapabilities"));
Dictionary <string, string> systemCapabilities = await HostContext.GetService <ICapabilitiesManager>().GetCapabilitiesAsync(agentSettings, CancellationToken.None);
_term.WriteLine(StringUtil.Loc("ConnectToServer"));
agent = await agentProvider.GetAgentAsync(agentSettings);
if (agent != null)
{
if (command.GetReplace())
{
// Update existing agent with new PublicKey, agent version and SystemCapabilities.
agent = UpdateExistingAgent(agent, publicKey, systemCapabilities);
try
{
agent = await agentProvider.UpdateAgentAsync(agentSettings, agent, command);
_term.WriteLine(StringUtil.Loc("AgentReplaced"));
break;
}
catch (Exception e) when(!command.Unattended)
{
_term.WriteError(e);
_term.WriteError(StringUtil.Loc("FailedToReplaceAgent"));
}
}
else if (command.Unattended)
{
// if not replace and it is unattended config.
agentProvider.ThrowTaskAgentExistException(agentSettings);
}
}
else
{
// Create a new agent.
agent = CreateNewAgent(agentSettings.AgentName, publicKey, systemCapabilities);
try
{
agent = await agentProvider.AddAgentAsync(agentSettings, agent, command);
_term.WriteLine(StringUtil.Loc("AgentAddedSuccessfully"));
break;
}
catch (Exception e) when(!command.Unattended)
{
_term.WriteError(e);
_term.WriteError(StringUtil.Loc("AddAgentFailed"));
}
}
}
// Add Agent Id to settings
agentSettings.AgentId = agent.Id;
// respect the serverUrl resolve by server.
// in case of agent configured using collection url instead of account url.
string agentServerUrl;
if (agent.Properties.TryGetValidatedValue <string>("ServerUrl", out agentServerUrl) &&
!string.IsNullOrEmpty(agentServerUrl))
{
Trace.Info($"Agent server url resolve by server: '{agentServerUrl}'.");
// we need make sure the Schema/Host/Port component of the url remain the same.
UriBuilder inputServerUrl = new UriBuilder(agentSettings.ServerUrl);
UriBuilder serverReturnedServerUrl = new UriBuilder(agentServerUrl);
if (Uri.Compare(inputServerUrl.Uri, serverReturnedServerUrl.Uri, UriComponents.SchemeAndServer, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) != 0)
{
inputServerUrl.Path = serverReturnedServerUrl.Path;
Trace.Info($"Replace server returned url's scheme://host:port component with user input server url's scheme://host:port: '{inputServerUrl.Uri.AbsoluteUri}'.");
agentSettings.ServerUrl = inputServerUrl.Uri.AbsoluteUri;
}
else
{
agentSettings.ServerUrl = agentServerUrl;
}
}
// See if the server supports our OAuth key exchange for credentials
if (agent.Authorization != null &&
agent.Authorization.ClientId != Guid.Empty &&
agent.Authorization.AuthorizationUrl != null)
{
// We use authorizationUrl as the oauth endpoint url by default.
// For TFS, we need make sure the Schema/Host/Port component of the oauth endpoint url also match configuration url. (Incase of customer's agent configure URL and TFS server public URL are different)
// Which means, we will keep use the original authorizationUrl in the VssOAuthJwtBearerClientCredential (authorizationUrl is the audience),
// But might have different Url in VssOAuthCredential (connection url)
// We can't do this for VSTS, since its SPS/TFS urls are different.
UriBuilder configServerUrl = new UriBuilder(agentSettings.ServerUrl);
UriBuilder oauthEndpointUrlBuilder = new UriBuilder(agent.Authorization.AuthorizationUrl);
if (!UrlUtil.IsHosted(configServerUrl.Uri.AbsoluteUri) &&
Uri.Compare(configServerUrl.Uri, oauthEndpointUrlBuilder.Uri, UriComponents.SchemeAndServer, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) != 0)
{
oauthEndpointUrlBuilder.Scheme = configServerUrl.Scheme;
oauthEndpointUrlBuilder.Host = configServerUrl.Host;
oauthEndpointUrlBuilder.Port = configServerUrl.Port;
Trace.Info($"Set oauth endpoint url's scheme://host:port component to match agent configure url's scheme://host:port: '{oauthEndpointUrlBuilder.Uri.AbsoluteUri}'.");
}
var credentialData = new CredentialData
{
Scheme = Constants.Configuration.OAuth,
Data =
{
{ "clientId", agent.Authorization.ClientId.ToString("D") },
{ "authorizationUrl", agent.Authorization.AuthorizationUrl.AbsoluteUri },
{ "oauthEndpointUrl", oauthEndpointUrlBuilder.Uri.AbsoluteUri },
},
};
// Save the negotiated OAuth credential data
_store.SaveCredential(credentialData);
}
else
{
switch (Constants.Agent.Platform)
{
case Constants.OSPlatform.OSX:
case Constants.OSPlatform.Linux:
// Save the provided admin cred for compat with previous agent.
_store.SaveCredential(credProvider.CredentialData);
break;
case Constants.OSPlatform.Windows:
// Not supported against TFS 2015.
_term.WriteError(StringUtil.Loc("Tfs2015NotSupported"));
return;
default:
throw new NotSupportedException();
}
}
// Testing agent connection, detect any protential connection issue, like local clock skew that cause OAuth token expired.
_term.WriteLine(StringUtil.Loc("TestAgentConnection"));
var credMgr = HostContext.GetService <ICredentialManager>();
VssCredentials credential = credMgr.LoadCredentials();
VssConnection conn = ApiUtil.CreateConnection(new Uri(agentSettings.ServerUrl), credential);
var agentSvr = HostContext.GetService <IAgentServer>();
try
{
await agentSvr.ConnectAsync(conn);
}
catch (VssOAuthTokenRequestException ex) when(ex.Message.Contains("Current server time is"))
{
// there are two exception messages server send that indicate clock skew.
// 1. The bearer token expired on {jwt.ValidTo}. Current server time is {DateTime.UtcNow}.
// 2. The bearer token is not valid until {jwt.ValidFrom}. Current server time is {DateTime.UtcNow}.
Trace.Error("Catch exception during test agent connection.");
Trace.Error(ex);
throw new Exception(StringUtil.Loc("LocalClockSkewed"));
}
// We will Combine() what's stored with root. Defaults to string a relative path
agentSettings.WorkFolder = command.GetWork();
// notificationPipeName for Hosted agent provisioner.
agentSettings.NotificationPipeName = command.GetNotificationPipeName();
agentSettings.NotificationSocketAddress = command.GetNotificationSocketAddress();
_store.SaveSettings(agentSettings);
if (saveProxySetting)
{
Trace.Info("Save proxy setting to disk.");
(vstsProxy as VstsAgentWebProxy).SaveProxySetting();
}
if (saveCertSetting)
{
Trace.Info("Save agent cert setting to disk.");
(agentCertManager as AgentCertificateManager).SaveCertificateSetting();
}
_term.WriteLine(StringUtil.Loc("SavedSettings", DateTime.UtcNow));
bool saveRuntimeOptions = false;
var runtimeOptions = new AgentRuntimeOptions();
#if OS_WINDOWS
if (command.GitUseSChannel)
{
saveRuntimeOptions = true;
runtimeOptions.GitUseSecureChannel = true;
}
#endif
if (saveRuntimeOptions)
{
Trace.Info("Save agent runtime options to disk.");
_store.SaveAgentRuntimeOptions(runtimeOptions);
}
#if OS_WINDOWS
// config windows service
bool runAsService = command.GetRunAsService();
if (runAsService)
{
Trace.Info("Configuring to run the agent as service");
var serviceControlManager = HostContext.GetService <IWindowsServiceControlManager>();
serviceControlManager.ConfigureService(agentSettings, command);
}
// config auto logon
else if (command.GetRunAsAutoLogon())
{
Trace.Info("Agent is going to run as process setting up the 'AutoLogon' capability for the agent.");
var autoLogonConfigManager = HostContext.GetService <IAutoLogonManager>();
await autoLogonConfigManager.ConfigureAsync(command);
//Important: The machine may restart if the autologon user is not same as the current user
//if you are adding code after this, keep that in mind
}
#elif OS_LINUX || OS_OSX
// generate service config script for OSX and Linux, GenerateScripts() will no-opt on windows.
var serviceControlManager = HostContext.GetService <ILinuxServiceControlManager>();
serviceControlManager.GenerateScripts(agentSettings);
#endif
}
public async Task RenewJobRequestAsync(int poolId, long requestId, Guid lockToken, TaskCompletionSource <int> firstJobRequestRenewed, CancellationToken token)
{
var agentServer = HostContext.GetService <IAgentServer>();
TaskAgentJobRequest request = null;
int firstRenewRetryLimit = 5;
int encounteringError = 0;
// renew lock during job running.
// stop renew only if cancellation token for lock renew task been signal or exception still happen after retry.
while (!token.IsCancellationRequested)
{
try
{
request = await agentServer.RenewAgentRequestAsync(poolId, requestId, lockToken, token);
Trace.Info($"Successfully renew job request {requestId}, job is valid till {request.LockedUntil.Value}");
if (!firstJobRequestRenewed.Task.IsCompleted)
{
// fire first renew succeed event.
firstJobRequestRenewed.TrySetResult(0);
}
if (encounteringError > 0)
{
encounteringError = 0;
agentServer.SetConnectionTimeout(AgentConnectionType.JobRequest, TimeSpan.FromSeconds(60));
HostContext.WritePerfCounter("JobRenewRecovered");
}
// renew again after 60 sec delay
await HostContext.Delay(TimeSpan.FromSeconds(60), token);
}
catch (TaskAgentJobNotFoundException)
{
// no need for retry. the job is not valid anymore.
Trace.Info($"TaskAgentJobNotFoundException received when renew job request {requestId}, job is no longer valid, stop renew job request.");
return;
}
catch (TaskAgentJobTokenExpiredException)
{
// no need for retry. the job is not valid anymore.
Trace.Info($"TaskAgentJobTokenExpiredException received renew job request {requestId}, job is no longer valid, stop renew job request.");
return;
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
// OperationCanceledException may caused by http timeout or _lockRenewalTokenSource.Cance();
// Stop renew only on cancellation token fired.
Trace.Info($"job renew has been canceled, stop renew job request {requestId}.");
return;
}
catch (Exception ex)
{
Trace.Error($"Catch exception during renew agent jobrequest {requestId}.");
Trace.Error(ex);
encounteringError++;
// retry
TimeSpan remainingTime = TimeSpan.Zero;
if (!firstJobRequestRenewed.Task.IsCompleted)
{
// retry 5 times every 10 sec for the first renew
if (firstRenewRetryLimit-- > 0)
{
remainingTime = TimeSpan.FromSeconds(10);
}
}
else
{
// retry till reach lockeduntil + 5 mins extra buffer.
remainingTime = request.LockedUntil.Value + TimeSpan.FromMinutes(5) - DateTime.UtcNow;
}
if (remainingTime > TimeSpan.Zero)
{
TimeSpan delayTime;
if (!firstJobRequestRenewed.Task.IsCompleted)
{
Trace.Info($"Retrying lock renewal for jobrequest {requestId}. The first job renew request has failed.");
delayTime = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
}
else
{
Trace.Info($"Retrying lock renewal for jobrequest {requestId}. Job is valid until {request.LockedUntil.Value}.");
if (encounteringError > 5)
{
delayTime = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30));
}
else
{
delayTime = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
}
}
// Re-establish connection to server in order to avoid affinity with server.
// Reduce connection timeout to 30 seconds (from 60s)
HostContext.WritePerfCounter("ResetJobRenewConnection");
await agentServer.RefreshConnectionAsync(AgentConnectionType.JobRequest, TimeSpan.FromSeconds(30));
try
{
// back-off before next retry.
await HostContext.Delay(delayTime, token);
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
Trace.Info($"job renew has been canceled, stop renew job request {requestId}.");
}
}
else
{
Trace.Info($"Lock renewal has run out of retry, stop renew lock for jobrequest {requestId}.");
HostContext.WritePerfCounter("JobRenewReachLimit");
return;
}
}
}
}
public async Task RunAsync()
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
ArgUtil.NotNull(ExecutionContext.Variables, nameof(ExecutionContext.Variables));
ArgUtil.NotNull(Task, nameof(Task));
var taskManager = HostContext.GetService <ITaskManager>();
var handlerFactory = HostContext.GetService <IHandlerFactory>();
// Set the task display name variable.
ExecutionContext.Variables.Set(Constants.Variables.Task.DisplayName, DisplayName);
// Load the task definition and choose the handler.
// TODO: Add a try catch here to give a better error message.
Definition definition = taskManager.Load(Task);
ArgUtil.NotNull(definition, nameof(definition));
// Print out task metadata
PrintTaskMetaData(definition);
ExecutionData currentExecution = null;
switch (Stage)
{
case JobRunStage.PreJob:
currentExecution = definition.Data?.PreJobExecution;
break;
case JobRunStage.Main:
currentExecution = definition.Data?.Execution;
break;
case JobRunStage.PostJob:
currentExecution = definition.Data?.PostJobExecution;
break;
}
;
if ((currentExecution?.All.Any(x => x is PowerShell3HandlerData)).Value &&
(currentExecution?.All.Any(x => x is PowerShellHandlerData && x.Platforms != null && x.Platforms.Contains("windows", StringComparer.OrdinalIgnoreCase))).Value)
{
// When task contains both PS and PS3 implementations, we will always prefer PS3 over PS regardless of the platform pinning.
Trace.Info("Ignore platform pinning for legacy PowerShell execution handler.");
var legacyPShandler = currentExecution?.All.Where(x => x is PowerShellHandlerData).FirstOrDefault();
legacyPShandler.Platforms = null;
}
HandlerData handlerData =
currentExecution?.All
.OrderBy(x => !x.PreferredOnCurrentPlatform()) // Sort true to false.
.ThenBy(x => x.Priority)
.FirstOrDefault();
if (handlerData == null)
{
throw new Exception(StringUtil.Loc("SupportedTaskHandlerNotFound"));
}
// Load the default input values from the definition.
Trace.Verbose("Loading default inputs.");
var inputs = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var input in (definition.Data?.Inputs ?? new TaskInputDefinition[0]))
{
string key = input?.Name?.Trim() ?? string.Empty;
if (!string.IsNullOrEmpty(key))
{
inputs[key] = input.DefaultValue?.Trim() ?? string.Empty;
}
}
// Merge the instance inputs.
Trace.Verbose("Loading instance inputs.");
foreach (var input in (Task.Inputs as IEnumerable <KeyValuePair <string, string> > ?? new KeyValuePair <string, string> [0]))
{
string key = input.Key?.Trim() ?? string.Empty;
if (!string.IsNullOrEmpty(key))
{
inputs[key] = input.Value?.Trim() ?? string.Empty;
}
}
// Expand the inputs.
Trace.Verbose("Expanding inputs.");
ExecutionContext.Variables.ExpandValues(target: inputs);
VarUtil.ExpandEnvironmentVariables(HostContext, target: inputs);
// Translate the server file path inputs to local paths.
foreach (var input in definition.Data?.Inputs ?? new TaskInputDefinition[0])
{
if (string.Equals(input.InputType, TaskInputType.FilePath, StringComparison.OrdinalIgnoreCase))
{
Trace.Verbose($"Translating file path input '{input.Name}': '{inputs[input.Name]}'");
inputs[input.Name] = TranslateFilePathInput(inputs[input.Name] ?? string.Empty);
Trace.Verbose($"Translated file path input '{input.Name}': '{inputs[input.Name]}'");
}
}
// Load the task environment.
Trace.Verbose("Loading task environment.");
var environment = new Dictionary <string, string>(VarUtil.EnvironmentVariableKeyComparer);
foreach (var env in (Task.Environment ?? new Dictionary <string, string>(0)))
{
string key = env.Key?.Trim() ?? string.Empty;
if (!string.IsNullOrEmpty(key))
{
environment[key] = env.Value?.Trim() ?? string.Empty;
}
}
// Expand the inputs.
Trace.Verbose("Expanding task environment.");
ExecutionContext.Variables.ExpandValues(target: environment);
VarUtil.ExpandEnvironmentVariables(HostContext, target: environment);
// Expand the handler inputs.
Trace.Verbose("Expanding handler inputs.");
VarUtil.ExpandValues(HostContext, source: inputs, target: handlerData.Inputs);
ExecutionContext.Variables.ExpandValues(target: handlerData.Inputs);
// Get each endpoint ID referenced by the task.
var endpointIds = new List <Guid>();
foreach (var input in definition.Data?.Inputs ?? new TaskInputDefinition[0])
{
if ((input.InputType ?? string.Empty).StartsWith("connectedService:", StringComparison.OrdinalIgnoreCase))
{
string inputKey = input?.Name?.Trim() ?? string.Empty;
string inputValue;
if (!string.IsNullOrEmpty(inputKey) &&
inputs.TryGetValue(inputKey, out inputValue) &&
!string.IsNullOrEmpty(inputValue))
{
foreach (string rawId in inputValue.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries))
{
Guid parsedId;
if (Guid.TryParse(rawId.Trim(), out parsedId) && parsedId != Guid.Empty)
{
endpointIds.Add(parsedId);
}
}
}
}
}
// Get the endpoints referenced by the task.
var endpoints = (ExecutionContext.Endpoints ?? new List <ServiceEndpoint>(0))
.Join(inner: endpointIds,
outerKeySelector: (ServiceEndpoint endpoint) => endpoint.Id,
innerKeySelector: (Guid endpointId) => endpointId,
resultSelector: (ServiceEndpoint endpoint, Guid endpointId) => endpoint)
.ToList();
// Add the system endpoint.
foreach (ServiceEndpoint endpoint in (ExecutionContext.Endpoints ?? new List <ServiceEndpoint>(0)))
{
if (string.Equals(endpoint.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase))
{
endpoints.Add(endpoint);
break;
}
}
// Get each secure file ID referenced by the task.
var secureFileIds = new List <Guid>();
foreach (var input in definition.Data?.Inputs ?? new TaskInputDefinition[0])
{
if (string.Equals(input.InputType ?? string.Empty, "secureFile", StringComparison.OrdinalIgnoreCase))
{
string inputKey = input?.Name?.Trim() ?? string.Empty;
string inputValue;
if (!string.IsNullOrEmpty(inputKey) &&
inputs.TryGetValue(inputKey, out inputValue) &&
!string.IsNullOrEmpty(inputValue))
{
foreach (string rawId in inputValue.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries))
{
Guid parsedId;
if (Guid.TryParse(rawId.Trim(), out parsedId) && parsedId != Guid.Empty)
{
secureFileIds.Add(parsedId);
}
}
}
}
}
// Get the endpoints referenced by the task.
var secureFiles = (ExecutionContext.SecureFiles ?? new List <SecureFile>(0))
.Join(inner: secureFileIds,
outerKeySelector: (SecureFile secureFile) => secureFile.Id,
innerKeySelector: (Guid secureFileId) => secureFileId,
resultSelector: (SecureFile secureFile, Guid secureFileId) => secureFile)
.ToList();
// Set output variables.
foreach (var outputVar in definition.Data?.OutputVariables ?? new OutputVariable[0])
{
if (outputVar != null && !string.IsNullOrEmpty(outputVar.Name))
{
ExecutionContext.OutputVariables.Add(outputVar.Name);
}
}
IStepHost stepHost;
// Get the container this task is going to use
if (ExecutionContext.Container != null)
{
// Make sure required container is already created.
ArgUtil.NotNullOrEmpty(ExecutionContext.Container.ContainerId, nameof(ExecutionContext.Container.ContainerId));
var containerStepHost = HostContext.CreateService <IContainerStepHost>();
containerStepHost.Container = ExecutionContext.Container;
stepHost = containerStepHost;
// Only node handler and powershell3 handler support running inside container.
if (!(handlerData is NodeHandlerData) &&
!(handlerData is PowerShell3HandlerData))
{
throw new NotSupportedException(String.Format("Task '{0}' is using legacy execution handler '{1}' which is not supported in container execution flow.", definition.Data.FriendlyName, handlerData.GetType().ToString()));
}
}
else
{
stepHost = HostContext.CreateService <IDefaultStepHost>();
}
// Create the handler.
IHandler handler = handlerFactory.Create(
ExecutionContext,
Task.Reference,
stepHost,
endpoints,
secureFiles,
handlerData,
inputs,
environment,
taskDirectory: definition.Directory,
filePathInputRootDirectory: TranslateFilePathInput(string.Empty));
// Run the task.
await handler.RunAsync();
}
public async Task UnconfigureAsync(CommandSettings command)
{
ArgUtil.Equal(RunMode.Normal, HostContext.RunMode, nameof(HostContext.RunMode));
string currentAction = string.Empty;
try
{
//stop, uninstall service and remove service config file
if (_store.IsServiceConfigured())
{
currentAction = StringUtil.Loc("UninstallingService");
_term.WriteLine(currentAction);
#if OS_WINDOWS
var serviceControlManager = HostContext.GetService <IWindowsServiceControlManager>();
serviceControlManager.UnconfigureService();
_term.WriteLine(StringUtil.Loc("Success") + currentAction);
#elif OS_LINUX
// unconfig system D service first
throw new Exception(StringUtil.Loc("UnconfigureServiceDService"));
#elif OS_OSX
// unconfig osx service first
throw new Exception(StringUtil.Loc("UnconfigureOSXService"));
#endif
}
else
{
#if OS_WINDOWS
//running as process, unconfigure autologon if it was configured
if (_store.IsAutoLogonConfigured())
{
currentAction = StringUtil.Loc("UnconfigAutologon");
_term.WriteLine(currentAction);
var autoLogonConfigManager = HostContext.GetService <IAutoLogonManager>();
autoLogonConfigManager.Unconfigure();
_term.WriteLine(StringUtil.Loc("Success") + currentAction);
}
else
{
Trace.Info("AutoLogon was not configured on the agent.");
}
#endif
}
//delete agent from the server
currentAction = StringUtil.Loc("UnregisteringAgent");
_term.WriteLine(currentAction);
bool isConfigured = _store.IsConfigured();
bool hasCredentials = _store.HasCredentials();
if (isConfigured && hasCredentials)
{
AgentSettings settings = _store.GetSettings();
var credentialManager = HostContext.GetService <ICredentialManager>();
// Get the credentials
var credProvider = GetCredentialProvider(command, settings.ServerUrl);
VssCredentials creds = credProvider.GetVssCredentials(HostContext);
Trace.Info("cred retrieved");
bool isDeploymentGroup = (settings.MachineGroupId > 0) || (settings.DeploymentGroupId > 0);
Trace.Info("Agent configured for deploymentGroup : {0}", isDeploymentGroup.ToString());
string agentType = isDeploymentGroup
? Constants.Agent.AgentConfigurationProvider.DeploymentAgentConfiguration
: Constants.Agent.AgentConfigurationProvider.BuildReleasesAgentConfiguration;
var extensionManager = HostContext.GetService <IExtensionManager>();
IConfigurationProvider agentProvider = (extensionManager.GetExtensions <IConfigurationProvider>()).FirstOrDefault(x => x.ConfigurationProviderType == agentType);
ArgUtil.NotNull(agentProvider, agentType);
await agentProvider.TestConnectionAsync(settings, creds);
TaskAgent agent = await agentProvider.GetAgentAsync(settings);
if (agent == null)
{
_term.WriteLine(StringUtil.Loc("Skipping") + currentAction);
}
else
{
await agentProvider.DeleteAgentAsync(settings);
_term.WriteLine(StringUtil.Loc("Success") + currentAction);
}
}
else
{
_term.WriteLine(StringUtil.Loc("MissingConfig"));
}
//delete credential config files
currentAction = StringUtil.Loc("DeletingCredentials");
_term.WriteLine(currentAction);
if (hasCredentials)
{
_store.DeleteCredential();
var keyManager = HostContext.GetService <IRSAKeyManager>();
keyManager.DeleteKey();
_term.WriteLine(StringUtil.Loc("Success") + currentAction);
}
else
{
_term.WriteLine(StringUtil.Loc("Skipping") + currentAction);
}
//delete settings config file
currentAction = StringUtil.Loc("DeletingSettings");
_term.WriteLine(currentAction);
if (isConfigured)
{
// delete proxy setting
(HostContext.GetService <IVstsAgentWebProxy>() as VstsAgentWebProxy).DeleteProxySetting();
// delete agent cert setting
(HostContext.GetService <IAgentCertificateManager>() as AgentCertificateManager).DeleteCertificateSetting();
// delete agent runtime option
_store.DeleteAgentRuntimeOptions();
_store.DeleteSettings();
_term.WriteLine(StringUtil.Loc("Success") + currentAction);
}
else
{
_term.WriteLine(StringUtil.Loc("Skipping") + currentAction);
}
}
catch (Exception)
{
_term.WriteLine(StringUtil.Loc("Failed") + currentAction);
throw;
}
}
public async Task <int> ExecuteCommand(CommandLineParser parser)
{
try
{
_inConfigStage = true;
_term.CancelKeyPress += CtrlCHandler;
// TODO Unit test to cover this logic
Trace.Info(nameof(ExecuteCommand));
var configManager = HostContext.GetService <IConfigurationManager>();
// command is not required, if no command it just starts and/or configures if not configured
// TODO: Invalid config prints usage
if (parser.Flags.Contains("help"))
{
PrintUsage();
return(0);
}
if (parser.Flags.Contains("version"))
{
_term.WriteLine(Constants.Agent.Version);
return(0);
}
if (parser.Flags.Contains("commit"))
{
_term.WriteLine(BuildConstants.Source.CommitHash);
return(0);
}
if (parser.IsCommand("unconfigure"))
{
Trace.Info("unconfigure");
// TODO: Unconfiure, remove config and exit
}
if (parser.IsCommand("run") && !configManager.IsConfigured())
{
Trace.Info("run");
_term.WriteError(StringUtil.Loc("AgentIsNotConfigured"));
PrintUsage();
return(1);
}
// unattend mode will not prompt for args if not supplied. Instead will error.
bool isUnattended = parser.Flags.Contains("unattended");
if (parser.IsCommand("configure"))
{
Trace.Info("configure");
try
{
await configManager.ConfigureAsync(parser.Args, parser.Flags, isUnattended);
return(0);
}
catch (Exception ex)
{
Trace.Error(ex);
_term.WriteError(ex.Message);
return(1);
}
}
if (parser.Flags.Contains("nostart"))
{
Trace.Info("No start option, exiting the agent");
return(0);
}
if (parser.IsCommand("run") && !configManager.IsConfigured())
{
throw new InvalidOperationException("CanNotRunAgent");
}
Trace.Info("Done evaluating commands");
bool alreadyConfigured = configManager.IsConfigured();
await configManager.EnsureConfiguredAsync();
_inConfigStage = false;
AgentSettings settings = configManager.LoadSettings();
if (parser.IsCommand("run") || !settings.RunAsService)
{
// Run the agent interactively
Trace.Verbose(
StringUtil.Format(
"Run command mentioned: {0}, run as service option mentioned:{1}",
parser.IsCommand("run"),
settings.RunAsService));
return(await RunAsync(TokenSource.Token, settings));
}
if (alreadyConfigured)
{
// This is helpful if the user tries to start the agent.listener which is already configured or running as service
// However user can execute the agent by calling the run command
// TODO: Should we check if the service is running and prompt user to start the service if its not already running?
_term.WriteLine(StringUtil.Loc("ConfiguredAsRunAsService", settings.ServiceName));
}
return(0);
}
finally
{
_term.CancelKeyPress -= CtrlCHandler;
}
}
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_term = HostContext.GetService <ITerminal>();
}
//create worker manager, create message listener and start listening to the queue
private async Task <int> RunAsync(CancellationToken token, AgentSettings settings)
{
Trace.Info(nameof(RunAsync));
// Load the settings.
_poolId = settings.PoolId;
var listener = HostContext.GetService <IMessageListener>();
if (!await listener.CreateSessionAsync(token))
{
return(1);
}
_term.WriteLine(StringUtil.Loc("ListenForJobs"));
_sessionId = listener.Session.SessionId;
TaskAgentMessage message = null;
try
{
using (var workerManager = HostContext.GetService <IWorkerManager>())
{
while (!token.IsCancellationRequested)
{
try
{
message = await listener.GetNextMessageAsync(token); //get next message
if (string.Equals(message.MessageType, AgentRefreshMessage.MessageType, StringComparison.OrdinalIgnoreCase))
{
Trace.Warning("Referesh message received, but not yet handled by agent implementation.");
}
else if (string.Equals(message.MessageType, JobRequestMessage.MessageType, StringComparison.OrdinalIgnoreCase))
{
var newJobMessage = JsonUtility.FromString <JobRequestMessage>(message.Body);
workerManager.Run(newJobMessage);
}
else if (string.Equals(message.MessageType, JobCancelMessage.MessageType, StringComparison.OrdinalIgnoreCase))
{
var cancelJobMessage = JsonUtility.FromString <JobCancelMessage>(message.Body);
workerManager.Cancel(cancelJobMessage);
}
}
finally
{
if (message != null)
{
//TODO: make sure we don't mask more important exception
await DeleteMessageAsync(message);
message = null;
}
}
}
}
}
finally
{
//TODO: make sure we don't mask more important exception
await listener.DeleteSessionAsync();
}
return(0);
}
public async Task <int> ExecuteCommand(CommandSettings command)
{
try
{
VssUtil.InitializeVssClientSettings(HostContext.UserAgents, HostContext.WebProxy);
_inConfigStage = true;
_completedCommand.Reset();
_term.CancelKeyPress += CtrlCHandler;
//register a SIGTERM handler
HostContext.Unloading += Runner_Unloading;
// TODO Unit test to cover this logic
Trace.Info(nameof(ExecuteCommand));
var configManager = HostContext.GetService <IConfigurationManager>();
// command is not required, if no command it just starts if configured
// TODO: Invalid config prints usage
if (command.Help)
{
PrintUsage(command);
return(Constants.Runner.ReturnCode.Success);
}
if (command.Version)
{
_term.WriteLine(BuildConstants.RunnerPackage.Version);
return(Constants.Runner.ReturnCode.Success);
}
if (command.Commit)
{
_term.WriteLine(BuildConstants.Source.CommitHash);
return(Constants.Runner.ReturnCode.Success);
}
// Configure runner prompt for args if not supplied
// Unattended configure mode will not prompt for args if not supplied and error on any missing or invalid value.
if (command.Configure)
{
try
{
await configManager.ConfigureAsync(command);
return(Constants.Runner.ReturnCode.Success);
}
catch (Exception ex)
{
Trace.Error(ex);
_term.WriteError(ex.Message);
return(Constants.Runner.ReturnCode.TerminatedError);
}
}
// remove config files, remove service, and exit
if (command.Remove)
{
try
{
await configManager.UnconfigureAsync(command);
return(Constants.Runner.ReturnCode.Success);
}
catch (Exception ex)
{
Trace.Error(ex);
_term.WriteError(ex.Message);
return(Constants.Runner.ReturnCode.TerminatedError);
}
}
_inConfigStage = false;
// warmup runner process (JIT/CLR)
// In scenarios where the runner is single use (used and then thrown away), the system provisioning the runner can call `Runner.Listener --warmup` before the machine is made available to the pool for use.
// this will optimizes the runner process startup time.
if (command.Warmup)
{
var binDir = HostContext.GetDirectory(WellKnownDirectory.Bin);
foreach (var assemblyFile in Directory.EnumerateFiles(binDir, "*.dll"))
{
try
{
Trace.Info($"Load assembly: {assemblyFile}.");
var assembly = Assembly.LoadFrom(assemblyFile);
var types = assembly.GetTypes();
foreach (Type loadedType in types)
{
try
{
Trace.Info($"Load methods: {loadedType.FullName}.");
var methods = loadedType.GetMethods(BindingFlags.DeclaredOnly | BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static);
foreach (var method in methods)
{
if (!method.IsAbstract && !method.ContainsGenericParameters)
{
Trace.Verbose($"Prepare method: {method.Name}.");
RuntimeHelpers.PrepareMethod(method.MethodHandle);
}
}
}
catch (Exception ex)
{
Trace.Error(ex);
}
}
}
catch (Exception ex)
{
Trace.Error(ex);
}
}
return(Constants.Runner.ReturnCode.Success);
}
RunnerSettings settings = configManager.LoadSettings();
var store = HostContext.GetService <IConfigurationStore>();
bool configuredAsService = store.IsServiceConfigured();
// Run runner
if (command.Run) // this line is current break machine provisioner.
{
// Error if runner not configured.
if (!configManager.IsConfigured())
{
_term.WriteError("Runner is not configured.");
PrintUsage(command);
return(Constants.Runner.ReturnCode.TerminatedError);
}
Trace.Verbose($"Configured as service: '{configuredAsService}'");
//Get the startup type of the runner i.e., autostartup, service, manual
StartupType startType;
var startupTypeAsString = command.GetStartupType();
if (string.IsNullOrEmpty(startupTypeAsString) && configuredAsService)
{
// We need try our best to make the startup type accurate
// The problem is coming from runner autoupgrade, which result an old version service host binary but a newer version runner binary
// At that time the servicehost won't pass --startuptype to Runner.Listener while the runner is actually running as service.
// We will guess the startup type only when the runner is configured as service and the guess will based on whether STDOUT/STDERR/STDIN been redirect or not
Trace.Info($"Try determine runner startup type base on console redirects.");
startType = (Console.IsErrorRedirected && Console.IsInputRedirected && Console.IsOutputRedirected) ? StartupType.Service : StartupType.Manual;
}
else
{
if (!Enum.TryParse(startupTypeAsString, true, out startType))
{
Trace.Info($"Could not parse the argument value '{startupTypeAsString}' for StartupType. Defaulting to {StartupType.Manual}");
startType = StartupType.Manual;
}
}
Trace.Info($"Set runner startup type - {startType}");
HostContext.StartupType = startType;
// Run the runner interactively or as service
return(await RunAsync(settings, command.RunOnce));
}
else
{
PrintUsage(command);
return(Constants.Runner.ReturnCode.Success);
}
}
finally
{
_term.CancelKeyPress -= CtrlCHandler;
HostContext.Unloading -= Runner_Unloading;
_completedCommand.Set();
}
}
private async Task DownloadArtifacts(IExecutionContext executionContext, List <AgentArtifactDefinition> agentArtifactDefinitions, string artifactsWorkingFolder)
{
Trace.Entering();
foreach (AgentArtifactDefinition agentArtifactDefinition in agentArtifactDefinitions)
{
// We don't need to check if its old style artifact anymore. All the build data has been fixed and all the build artifact has Alias now.
ArgUtil.NotNullOrEmpty(agentArtifactDefinition.Alias, nameof(agentArtifactDefinition.Alias));
var extensionManager = HostContext.GetService <IExtensionManager>();
IArtifactExtension extension = (extensionManager.GetExtensions <IArtifactExtension>()).FirstOrDefault(x => agentArtifactDefinition.ArtifactType == x.ArtifactType);
if (extension == null)
{
throw new InvalidOperationException(StringUtil.Loc("RMArtifactTypeNotSupported"));
}
Trace.Info($"Found artifact extension of type {extension.ArtifactType}");
executionContext.Output(StringUtil.Loc("RMStartArtifactsDownload"));
ArtifactDefinition artifactDefinition = ConvertToArtifactDefinition(agentArtifactDefinition, executionContext, extension);
executionContext.Output(StringUtil.Loc("RMArtifactDownloadBegin", agentArtifactDefinition.Alias));
executionContext.Output(StringUtil.Loc("RMDownloadArtifactType", agentArtifactDefinition.ArtifactType));
// Get the local path where this artifact should be downloaded.
string downloadFolderPath = Path.GetFullPath(Path.Combine(artifactsWorkingFolder, agentArtifactDefinition.Alias ?? string.Empty));
// Create the directory if it does not exist.
if (!Directory.Exists(downloadFolderPath))
{
// TODO: old windows agent has a directory cache, verify and implement it if its required.
Directory.CreateDirectory(downloadFolderPath);
executionContext.Output(StringUtil.Loc("RMArtifactFolderCreated", downloadFolderPath));
}
// download the artifact to this path.
RetryExecutor retryExecutor = new RetryExecutor();
retryExecutor.ShouldRetryAction = (ex) =>
{
executionContext.Output(StringUtil.Loc("RMErrorDuringArtifactDownload", ex));
bool retry = true;
if (ex is ArtifactDownloadException)
{
retry = false;
}
else
{
executionContext.Output(StringUtil.Loc("RMRetryingArtifactDownload"));
}
return(retry);
};
await retryExecutor.ExecuteAsync(
async() =>
{
//TODO:SetAttributesToNormal
var releaseFileSystemManager = HostContext.GetService <IReleaseFileSystemManager>();
releaseFileSystemManager.CleanupDirectory(downloadFolderPath, executionContext.CancellationToken);
if (agentArtifactDefinition.ArtifactType == AgentArtifactType.GitHub ||
agentArtifactDefinition.ArtifactType == AgentArtifactType.TFGit ||
agentArtifactDefinition.ArtifactType == AgentArtifactType.Tfvc)
{
throw new NotImplementedException();
}
else
{
await extension.DownloadAsync(executionContext, artifactDefinition, downloadFolderPath);
}
});
executionContext.Output(StringUtil.Loc("RMArtifactDownloadFinished", agentArtifactDefinition.Alias));
}
executionContext.Output(StringUtil.Loc("RMArtifactsDownloadFinished"));
}
private async Task RunAsync(Pipelines.AgentJobRequestMessage message, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken, CancellationToken workerCancelTimeoutKillToken)
{
if (previousJobDispatch != null)
{
Trace.Verbose($"Make sure the previous job request {previousJobDispatch.JobId} has successfully finished on worker.");
await EnsureDispatchFinished(previousJobDispatch);
}
else
{
Trace.Verbose($"This is the first job request.");
}
var term = HostContext.GetService <ITerminal>();
term.WriteLine(StringUtil.Loc("RunningJob", DateTime.UtcNow, message.JobDisplayName));
// first job request renew succeed.
TaskCompletionSource <int> firstJobRequestRenewed = new TaskCompletionSource <int>();
var notification = HostContext.GetService <IJobNotification>();
// lock renew cancellation token.
using (var lockRenewalTokenSource = new CancellationTokenSource())
using (var workerProcessCancelTokenSource = new CancellationTokenSource())
{
long requestId = message.RequestId;
Guid lockToken = Guid.Empty; // lockToken has never been used, keep this here of compat
// start renew job request
Trace.Info($"Start renew job request {requestId} for job {message.JobId}.");
Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, firstJobRequestRenewed, lockRenewalTokenSource.Token);
// wait till first renew succeed or job request is canceled
// not even start worker if the first renew fail
await Task.WhenAny(firstJobRequestRenewed.Task, renewJobRequest, Task.Delay(-1, jobRequestCancellationToken));
if (renewJobRequest.IsCompleted)
{
// renew job request task complete means we run out of retry for the first job request renew.
Trace.Info($"Unable to renew job request for job {message.JobId} for the first time, stop dispatching job to worker.");
return;
}
if (jobRequestCancellationToken.IsCancellationRequested)
{
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request with result Cancelled
await CompleteJobRequestAsync(_poolId, message, lockToken, TaskResult.Canceled);
return;
}
HostContext.WritePerfCounter($"JobRequestRenewed_{requestId.ToString()}");
Task <int> workerProcessTask = null;
object _outputLock = new object();
List <string> workerOutput = new List <string>();
using (var processChannel = HostContext.CreateService <IProcessChannel>())
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
// Start the process channel.
// It's OK if StartServer bubbles an execption after the worker process has already started.
// The worker will shutdown after 30 seconds if it hasn't received the job message.
processChannel.StartServer(
// Delegate to start the child process.
startProcess: (string pipeHandleOut, string pipeHandleIn) =>
{
// Validate args.
ArgUtil.NotNullOrEmpty(pipeHandleOut, nameof(pipeHandleOut));
ArgUtil.NotNullOrEmpty(pipeHandleIn, nameof(pipeHandleIn));
if (HostContext.RunMode == RunMode.Normal)
{
// Save STDOUT from worker, worker will use STDOUT report unhandle exception.
processInvoker.OutputDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stdout)
{
if (!string.IsNullOrEmpty(stdout.Data))
{
lock (_outputLock)
{
workerOutput.Add(stdout.Data);
}
}
};
// Save STDERR from worker, worker will use STDERR on crash.
processInvoker.ErrorDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stderr)
{
if (!string.IsNullOrEmpty(stderr.Data))
{
lock (_outputLock)
{
workerOutput.Add(stderr.Data);
}
}
};
}
else if (HostContext.RunMode == RunMode.Local)
{
processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) => Console.WriteLine(e.Data);
processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) => Console.WriteLine(e.Data);
}
// Start the child process.
HostContext.WritePerfCounter("StartingWorkerProcess");
var assemblyDirectory = HostContext.GetDirectory(WellKnownDirectory.Bin);
string workerFileName = Path.Combine(assemblyDirectory, _workerProcessName);
workerProcessTask = processInvoker.ExecuteAsync(
workingDirectory: assemblyDirectory,
fileName: workerFileName,
arguments: "spawnclient " + pipeHandleOut + " " + pipeHandleIn,
environment: null,
requireExitCodeZero: false,
outputEncoding: null,
killProcessOnCancel: true,
redirectStandardIn: null,
inheritConsoleHandler: false,
keepStandardInOpen: false,
highPriorityProcess: true,
cancellationToken: workerProcessCancelTokenSource.Token);
});
// Send the job request message.
// Kill the worker process if sending the job message times out. The worker
// process may have successfully received the job message.
try
{
Trace.Info($"Send job request message to worker for job {message.JobId}.");
HostContext.WritePerfCounter($"AgentSendingJobToWorker_{message.JobId}");
using (var csSendJobRequest = new CancellationTokenSource(_channelTimeout))
{
await processChannel.SendAsync(
messageType : MessageType.NewJobRequest,
body : JsonUtility.ToString(message),
cancellationToken : csSendJobRequest.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
// timeout 30 sec. kill worker.
Trace.Info($"Job request message sending for job {message.JobId} been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// not finish the job request since the job haven't run on worker at all, we will not going to set a result to server.
return;
}
// we get first jobrequest renew succeed and start the worker process with the job message.
// send notification to machine provisioner.
var systemConnection = message.Resources.Endpoints.SingleOrDefault(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
var accessToken = systemConnection?.Authorization?.Parameters["AccessToken"];
VariableValue identifier = new VariableValue("0");
VariableValue definitionId = new VariableValue("0");
if (message.Plan.PlanType == "Build")
{
message.Variables.TryGetValue("build.buildId", out identifier);
message.Variables.TryGetValue("system.definitionId", out definitionId);
}
else if (message.Plan.PlanType == "Release")
{
message.Variables.TryGetValue("release.deploymentId", out identifier);
message.Variables.TryGetValue("release.definitionId", out definitionId);
}
await notification.JobStarted(message.JobId, accessToken, systemConnection.Url, message.Plan.PlanId, identifier.Value, definitionId.Value, message.Plan.PlanType);
HostContext.WritePerfCounter($"SentJobToWorker_{requestId.ToString()}");
try
{
TaskResult resultOnAbandonOrCancel = TaskResult.Succeeded;
// wait for renewlock, worker process or cancellation token been fired.
var completedTask = await Task.WhenAny(renewJobRequest, workerProcessTask, Task.Delay(-1, jobRequestCancellationToken));
if (completedTask == workerProcessTask)
{
// worker finished successfully, complete job request with result, attach unhandled exception reported by worker, stop renew lock, job has finished.
int returnCode = await workerProcessTask;
Trace.Info($"Worker finished for job {message.JobId}. Code: " + returnCode);
string detailInfo = null;
if (!TaskResultUtil.IsValidReturnCode(returnCode))
{
detailInfo = string.Join(Environment.NewLine, workerOutput);
Trace.Info($"Return code {returnCode} indicate worker encounter an unhandled exception or app crash, attach worker stdout/stderr to JobRequest result.");
await LogWorkerProcessUnhandledException(message, detailInfo);
}
TaskResult result = TaskResultUtil.TranslateFromReturnCode(returnCode);
Trace.Info($"finish job request for job {message.JobId} with result: {result}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobDisplayName, result));
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request
await CompleteJobRequestAsync(_poolId, message, lockToken, result, detailInfo);
// print out unhandled exception happened in worker after we complete job request.
// when we run out of disk space, report back to server has higher priority.
if (!string.IsNullOrEmpty(detailInfo))
{
Trace.Error("Unhandled exception happened in worker:");
Trace.Error(detailInfo);
}
return;
}
else if (completedTask == renewJobRequest)
{
resultOnAbandonOrCancel = TaskResult.Abandoned;
}
else
{
resultOnAbandonOrCancel = TaskResult.Canceled;
}
// renew job request completed or job request cancellation token been fired for RunAsync(jobrequestmessage)
// cancel worker gracefully first, then kill it after worker cancel timeout
try
{
Trace.Info($"Send job cancellation message to worker for job {message.JobId}.");
using (var csSendCancel = new CancellationTokenSource(_channelTimeout))
{
var messageType = MessageType.CancelRequest;
if (HostContext.AgentShutdownToken.IsCancellationRequested)
{
switch (HostContext.AgentShutdownReason)
{
case ShutdownReason.UserCancelled:
messageType = MessageType.AgentShutdown;
break;
case ShutdownReason.OperatingSystemShutdown:
messageType = MessageType.OperatingSystemShutdown;
break;
}
}
await processChannel.SendAsync(
messageType : messageType,
body : string.Empty,
cancellationToken : csSendCancel.Token);
}
}
catch (OperationCanceledException)
{
// message send been cancelled.
Trace.Info($"Job cancel message sending for job {message.JobId} been cancelled, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
}
// wait worker to exit
// if worker doesn't exit within timeout, then kill worker.
completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, workerCancelTimeoutKillToken));
// worker haven't exit within cancellation timeout.
if (completedTask != workerProcessTask)
{
Trace.Info($"worker process for job {message.JobId} haven't exit within cancellation timout, kill running worker.");
workerProcessCancelTokenSource.Cancel();
try
{
await workerProcessTask;
}
catch (OperationCanceledException)
{
Trace.Info("worker process has been killed.");
}
}
Trace.Info($"finish job request for job {message.JobId} with result: {resultOnAbandonOrCancel}");
term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobDisplayName, resultOnAbandonOrCancel));
// complete job request with cancel result, stop renew lock, job has finished.
Trace.Info($"Stop renew job request for job {message.JobId}.");
// stop renew lock
lockRenewalTokenSource.Cancel();
// renew job request should never blows up.
await renewJobRequest;
// complete job request
await CompleteJobRequestAsync(_poolId, message, lockToken, resultOnAbandonOrCancel);
}
finally
{
// This should be the last thing to run so we don't notify external parties until actually finished
await notification.JobCompleted(message.JobId);
}
}
}
}
public async Task <int> RunAsync(string pipeIn, string pipeOut)
{
// Validate args.
ArgUtil.NotNullOrEmpty(pipeIn, nameof(pipeIn));
ArgUtil.NotNullOrEmpty(pipeOut, nameof(pipeOut));
var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>();
var agentCertManager = HostContext.GetService <IAgentCertificateManager>();
VssUtil.InitializeVssClientSettings(HostContext.UserAgent, agentWebProxy.WebProxy, agentCertManager.VssClientCertificateManager);
var jobRunner = HostContext.CreateService <IJobRunner>();
using (var channel = HostContext.CreateService <IProcessChannel>())
using (var jobRequestCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(HostContext.AgentShutdownToken))
using (var channelTokenSource = new CancellationTokenSource())
{
// Start the channel.
channel.StartClient(pipeIn, pipeOut);
// Wait for up to 30 seconds for a message from the channel.
HostContext.WritePerfCounter("WorkerWaitingForJobMessage");
Trace.Info("Waiting to receive the job message from the channel.");
WorkerMessage channelMessage;
using (var csChannelMessage = new CancellationTokenSource(_workerStartTimeout))
{
channelMessage = await channel.ReceiveAsync(csChannelMessage.Token);
}
// Deserialize the job message.
Trace.Info("Message received.");
ArgUtil.Equal(MessageType.NewJobRequest, channelMessage.MessageType, nameof(channelMessage.MessageType));
ArgUtil.NotNullOrEmpty(channelMessage.Body, nameof(channelMessage.Body));
var jobMessage = JsonUtility.FromString <Pipelines.AgentJobRequestMessage>(channelMessage.Body);
ArgUtil.NotNull(jobMessage, nameof(jobMessage));
HostContext.WritePerfCounter($"WorkerJobMessageReceived_{jobMessage.RequestId.ToString()}");
// Initialize the secret masker and set the thread culture.
InitializeSecretMasker(jobMessage);
SetCulture(jobMessage);
// Start the job.
Trace.Info($"Job message:{Environment.NewLine} {StringUtil.ConvertToJson(WorkerUtilities.ScrubPiiData(jobMessage))}");
Task <TaskResult> jobRunnerTask = jobRunner.RunAsync(jobMessage, jobRequestCancellationToken.Token);
// Start listening for a cancel message from the channel.
Trace.Info("Listening for cancel message from the channel.");
Task <WorkerMessage> channelTask = channel.ReceiveAsync(channelTokenSource.Token);
// Wait for one of the tasks to complete.
Trace.Info("Waiting for the job to complete or for a cancel message from the channel.");
Task.WaitAny(jobRunnerTask, channelTask);
// Handle if the job completed.
if (jobRunnerTask.IsCompleted)
{
Trace.Info("Job completed.");
channelTokenSource.Cancel(); // Cancel waiting for a message from the channel.
return(TaskResultUtil.TranslateToReturnCode(await jobRunnerTask));
}
// Otherwise a cancel message was received from the channel.
Trace.Info("Cancellation/Shutdown message received.");
channelMessage = await channelTask;
switch (channelMessage.MessageType)
{
case MessageType.CancelRequest:
jobRequestCancellationToken.Cancel(); // Expire the host cancellation token.
break;
case MessageType.AgentShutdown:
HostContext.ShutdownAgent(ShutdownReason.UserCancelled);
break;
case MessageType.OperatingSystemShutdown:
HostContext.ShutdownAgent(ShutdownReason.OperatingSystemShutdown);
break;
default:
throw new ArgumentOutOfRangeException(nameof(channelMessage.MessageType), channelMessage.MessageType, nameof(channelMessage.MessageType));
}
// Await the job.
return(TaskResultUtil.TranslateToReturnCode(await jobRunnerTask));
}
}
// Return code definition: (this will be used by service host to determine whether it will re-launch agent.listener)
// 0: Agent exit
// 1: Terminate failure
// 2: Retriable failure
// 3: Exit for self update
public async static Task<int> MainAsync(string[] args)
{
using (HostContext context = new HostContext("Agent"))
{
s_trace = context.GetTrace("AgentProcess");
s_trace.Info($"Agent is built for {Constants.Agent.Platform} - {BuildConstants.AgentPackage.PackageName}.");
s_trace.Info($"RuntimeInformation: {RuntimeInformation.OSDescription}.");
// Validate the binaries intended for one OS are not running on a different OS.
switch (Constants.Agent.Platform)
{
case Constants.OSPlatform.Linux:
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
Console.WriteLine(StringUtil.Loc("NotLinux"));
return Constants.Agent.ReturnCode.TerminatedError;
}
break;
case Constants.OSPlatform.OSX:
if (!RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
Console.WriteLine(StringUtil.Loc("NotOSX"));
return Constants.Agent.ReturnCode.TerminatedError;
}
break;
case Constants.OSPlatform.Windows:
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
Console.WriteLine(StringUtil.Loc("NotWindows"));
return Constants.Agent.ReturnCode.TerminatedError;
}
break;
default:
Console.WriteLine(StringUtil.Loc("PlatformNotSupport", RuntimeInformation.OSDescription, Constants.Agent.Platform.ToString()));
return Constants.Agent.ReturnCode.TerminatedError;
}
int rc = Constants.Agent.ReturnCode.Success;
try
{
s_trace.Info($"Version: {Constants.Agent.Version}");
s_trace.Info($"Commit: {BuildConstants.Source.CommitHash}");
s_trace.Info($"Culture: {CultureInfo.CurrentCulture.Name}");
s_trace.Info($"UI Culture: {CultureInfo.CurrentUICulture.Name}");
//
// TODO (bryanmac): Need VsoAgent.exe compat shim for SCM
// That shim will also provide a compat arg parse
// and translate / to -- etc...
//
// Parse the command line args.
var command = new CommandSettings(context, args);
s_trace.Info("Arguments parsed");
// Defer to the Agent class to execute the command.
IAgent agent = context.GetService<IAgent>();
using (agent.TokenSource = new CancellationTokenSource())
{
try
{
rc = await agent.ExecuteCommand(command);
}
catch (OperationCanceledException) when (agent.TokenSource.IsCancellationRequested)
{
s_trace.Info("Agent execution been cancelled.");
}
}
}
catch (Exception e)
{
Console.Error.WriteLine(StringUtil.Format("An error occured. {0}", e.Message));
s_trace.Error(e);
rc = Constants.Agent.ReturnCode.RetryableError;
}
return rc;
}
}
public async Task RunAsync()
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
ArgUtil.NotNull(Inputs, nameof(Inputs));
ArgUtil.Directory(TaskDirectory, nameof(TaskDirectory));
// Resolve the target script.
string target = GetTarget();
ArgUtil.NotNullOrEmpty(target, nameof(target));
string scriptFile = Path.Combine(TaskDirectory, target);
ArgUtil.File(scriptFile, nameof(scriptFile));
// Determine the working directory.
string workingDirectory = GetWorkingDirectory();
if (String.IsNullOrEmpty(workingDirectory))
{
workingDirectory = Path.GetDirectoryName(scriptFile);
}
else
{
if (!Directory.Exists(workingDirectory))
{
Directory.CreateDirectory(workingDirectory);
}
}
// Copy the OM binaries into the legacy host folder.
ExecutionContext.Output(StringUtil.Loc("PrepareTaskExecutionHandler"));
IOUtil.CopyDirectory(
source: HostContext.GetDirectory(WellKnownDirectory.ServerOM),
target: HostContext.GetDirectory(WellKnownDirectory.LegacyPSHost),
cancellationToken: ExecutionContext.CancellationToken);
Trace.Info("Finished copying files.");
// Add the legacy ps host environment variables.
AddLegacyHostEnvironmentVariables(scriptFile: scriptFile, workingDirectory: workingDirectory);
AddPrependPathToEnvironment();
// Add proxy setting to LegacyVSTSPowerShellHost.exe.config
var agentProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(agentProxy.ProxyAddress))
{
AddProxySetting(agentProxy);
}
// Invoke the process.
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += OnDataReceived;
processInvoker.ErrorDataReceived += OnDataReceived;
try
{
String vstsPSHostExe = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.LegacyPSHost), "LegacyVSTSPowerShellHost.exe");
Int32 exitCode = await processInvoker.ExecuteAsync(workingDirectory : workingDirectory,
fileName : vstsPSHostExe,
arguments : "",
environment : Environment,
requireExitCodeZero : false,
outputEncoding : null,
killProcessOnCancel : false,
enhancedProcessesCleanup : ExecutionContext.Variables.GetBoolean("process.clean") ?? false,
cancellationToken : ExecutionContext.CancellationToken);
// the exit code from vstsPSHost.exe indicate how many error record we get during execution
// -1 exit code means infrastructure failure of Host itself.
// this is to match current handler's logic.
if (exitCode > 0)
{
if (ExecutionContext.Result != null)
{
ExecutionContext.Debug($"Task result already set. Not failing due to error count ({exitCode}).");
}
else
{
// We fail task and add issue.
ExecutionContext.Result = TaskResult.Failed;
ExecutionContext.Error(StringUtil.Loc("PSScriptError", exitCode));
}
}
else if (exitCode < 0)
{
// We fail task and add issue.
ExecutionContext.Result = TaskResult.Failed;
ExecutionContext.Error(StringUtil.Loc("VSTSHostNonZeroReturn", exitCode));
}
}
finally
{
processInvoker.OutputDataReceived -= OnDataReceived;
processInvoker.ErrorDataReceived -= OnDataReceived;
}
}
}
private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, Pipelines.ActionStep repositoryAction)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
var repositoryReference = repositoryAction.Reference as Pipelines.RepositoryPathReference;
ArgUtil.NotNull(repositoryReference, nameof(repositoryReference));
if (string.Equals(repositoryReference.RepositoryType, Pipelines.PipelineConstants.SelfAlias, StringComparison.OrdinalIgnoreCase))
{
Trace.Info($"Repository action is in 'self' repository.");
return;
}
if (!string.Equals(repositoryReference.RepositoryType, Pipelines.RepositoryTypes.GitHub, StringComparison.OrdinalIgnoreCase))
{
throw new NotSupportedException(repositoryReference.RepositoryType);
}
ArgUtil.NotNullOrEmpty(repositoryReference.Name, nameof(repositoryReference.Name));
ArgUtil.NotNullOrEmpty(repositoryReference.Ref, nameof(repositoryReference.Ref));
string destDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), repositoryReference.Name.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), repositoryReference.Ref);
string watermarkFile = destDirectory + ".completed";
if (File.Exists(watermarkFile))
{
executionContext.Debug($"Action '{repositoryReference.Name}@{repositoryReference.Ref}' already downloaded at '{destDirectory}'.");
return;
}
else
{
// make sure we get a clean folder ready to use.
IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken);
Directory.CreateDirectory(destDirectory);
executionContext.Output($"Download action repository '{repositoryReference.Name}@{repositoryReference.Ref}'");
}
#if OS_WINDOWS
string archiveLink = $"https://api.github.com/repos/{repositoryReference.Name}/zipball/{repositoryReference.Ref}";
#else
string archiveLink = $"https://api.github.com/repos/{repositoryReference.Name}/tarball/{repositoryReference.Ref}";
#endif
Trace.Info($"Download archive '{archiveLink}' to '{destDirectory}'.");
//download and extract action in a temp folder and rename it on success
string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), "_temp_" + Guid.NewGuid());
Directory.CreateDirectory(tempDirectory);
#if OS_WINDOWS
string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.zip");
#else
string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.tar.gz");
#endif
Trace.Info($"Save archive '{archiveLink}' into {archiveFile}.");
try
{
int retryCount = 0;
// Allow up to 20 * 60s for any action to be downloaded from github graph.
int timeoutSeconds = 20 * 60;
while (retryCount < 3)
{
using (var actionDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
using (var actionDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(actionDownloadTimeout.Token, executionContext.CancellationToken))
{
try
{
//open zip stream in async mode
using (FileStream fs = new FileStream(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true))
using (var httpClientHandler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(httpClientHandler))
{
var configurationStore = HostContext.GetService <IConfigurationStore>();
var isHostedServer = configurationStore.GetSettings().IsHostedServer;
if (isHostedServer)
{
var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN");
if (string.IsNullOrEmpty(authToken))
{
// TODO: Deprecate the PREVIEW_ACTION_TOKEN
authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN");
}
if (!string.IsNullOrEmpty(authToken))
{
HostContext.SecretMasker.AddValue(authToken);
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"PAT:{authToken}"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
}
else
{
var accessToken = executionContext.GetGitHubContext("token");
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
}
}
else
{
// Intentionally empty. Temporary for GHES alpha release, download from dotcom unauthenticated.
}
httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents);
using (var result = await httpClient.GetStreamAsync(archiveLink))
{
await result.CopyToAsync(fs, _defaultCopyBufferSize, actionDownloadCancellation.Token);
await fs.FlushAsync(actionDownloadCancellation.Token);
// download succeed, break out the retry loop.
break;
}
}
}
catch (OperationCanceledException) when(executionContext.CancellationToken.IsCancellationRequested)
{
Trace.Info($"Action download has been cancelled.");
throw;
}
catch (Exception ex) when(retryCount < 2)
{
retryCount++;
Trace.Error($"Fail to download archive '{archiveLink}' -- Attempt: {retryCount}");
Trace.Error(ex);
if (actionDownloadTimeout.Token.IsCancellationRequested)
{
// action download didn't finish within timeout
executionContext.Warning($"Action '{archiveLink}' didn't finish download within {timeoutSeconds} seconds.");
}
else
{
executionContext.Warning($"Failed to download action '{archiveLink}'. Error {ex.Message}");
}
}
}
if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("_GITHUB_ACTION_DOWNLOAD_NO_BACKOFF")))
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30));
executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
ArgUtil.NotNullOrEmpty(archiveFile, nameof(archiveFile));
executionContext.Debug($"Download '{archiveLink}' to '{archiveFile}'");
var stagingDirectory = Path.Combine(tempDirectory, "_staging");
Directory.CreateDirectory(stagingDirectory);
#if OS_WINDOWS
ZipFile.ExtractToDirectory(archiveFile, stagingDirectory);
#else
string tar = WhichUtil.Which("tar", require: true, trace: Trace);
// tar -xzf
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Info(args.Data);
}
});
processInvoker.ErrorDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Error(args.Data);
}
});
int exitCode = await processInvoker.ExecuteAsync(stagingDirectory, tar, $"-xzf \"{archiveFile}\"", null, executionContext.CancellationToken);
if (exitCode != 0)
{
throw new NotSupportedException($"Can't use 'tar -xzf' extract archive file: {archiveFile}. return code: {exitCode}.");
}
}
#endif
// repository archive from github always contains a nested folder
var subDirectories = new DirectoryInfo(stagingDirectory).GetDirectories();
if (subDirectories.Length != 1)
{
throw new InvalidOperationException($"'{archiveFile}' contains '{subDirectories.Length}' directories");
}
else
{
executionContext.Debug($"Unwrap '{subDirectories[0].Name}' to '{destDirectory}'");
IOUtil.CopyDirectory(subDirectories[0].FullName, destDirectory, executionContext.CancellationToken);
}
Trace.Verbose("Create watermark file indicate action download succeed.");
File.WriteAllText(watermarkFile, DateTime.UtcNow.ToString());
executionContext.Debug($"Archive '{archiveFile}' has been unzipped into '{destDirectory}'.");
Trace.Info("Finished getting action repository.");
}
finally
{
try
{
//if the temp folder wasn't moved -> wipe it
if (Directory.Exists(tempDirectory))
{
Trace.Verbose("Deleting action temp folder: {0}", tempDirectory);
IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast.
}
}
catch (Exception ex)
{
//it is not critical if we fail to delete the temp folder
Trace.Warning("Failed to delete temp folder '{0}'. Exception: {1}", tempDirectory, ex);
}
}
}
// Return code definition: (this will be used by service host to determine whether it will re-launch agent.listener)
// 0: Agent exit
// 1: Terminate failure
// 2: Retriable failure
// 3: Exit for self update
public async static Task <int> MainAsync(string[] args)
{
using (HostContext context = new HostContext("Agent"))
{
s_trace = context.GetTrace("AgentProcess");
s_trace.Info($"Agent is built for {Constants.Agent.Platform} - {BuildConstants.AgentPackage.PackageName}.");
s_trace.Info($"RuntimeInformation: {RuntimeInformation.OSDescription}.");
// Validate the binaries intended for one OS are not running on a different OS.
switch (Constants.Agent.Platform)
{
case Constants.OSPlatform.Linux:
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
Console.WriteLine(StringUtil.Loc("NotLinux"));
return(Constants.Agent.ReturnCode.TerminatedError);
}
break;
case Constants.OSPlatform.OSX:
if (!RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
Console.WriteLine(StringUtil.Loc("NotOSX"));
return(Constants.Agent.ReturnCode.TerminatedError);
}
break;
case Constants.OSPlatform.Windows:
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
Console.WriteLine(StringUtil.Loc("NotWindows"));
return(Constants.Agent.ReturnCode.TerminatedError);
}
break;
default:
Console.WriteLine(StringUtil.Loc("PlatformNotSupport", RuntimeInformation.OSDescription, Constants.Agent.Platform.ToString()));
return(Constants.Agent.ReturnCode.TerminatedError);
}
int rc = Constants.Agent.ReturnCode.Success;
try
{
s_trace.Info($"Version: {Constants.Agent.Version}");
s_trace.Info($"Commit: {BuildConstants.Source.CommitHash}");
s_trace.Info($"Culture: {CultureInfo.CurrentCulture.Name}");
s_trace.Info($"UI Culture: {CultureInfo.CurrentUICulture.Name}");
//
// TODO (bryanmac): Need VsoAgent.exe compat shim for SCM
// That shim will also provide a compat arg parse
// and translate / to -- etc...
//
// Parse the command line args.
var command = new CommandSettings(context, args);
s_trace.Info("Arguments parsed");
// Defer to the Agent class to execute the command.
IAgent agent = context.GetService <IAgent>();
using (agent.TokenSource = new CancellationTokenSource())
{
try
{
rc = await agent.ExecuteCommand(command);
}
catch (OperationCanceledException) when(agent.TokenSource.IsCancellationRequested)
{
s_trace.Info("Agent execution been cancelled.");
}
}
}
catch (Exception e)
{
Console.Error.WriteLine(StringUtil.Format("An error occured. {0}", e.Message));
s_trace.Error(e);
rc = Constants.Agent.ReturnCode.RetryableError;
}
return(rc);
}
}
public void InitializeJob(JobRequestMessage message, CancellationToken token)
{
// Validation
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Environment, nameof(message.Environment));
ArgUtil.NotNull(message.Environment.SystemConnection, nameof(message.Environment.SystemConnection));
ArgUtil.NotNull(message.Environment.Endpoints, nameof(message.Environment.Endpoints));
ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables));
ArgUtil.NotNull(message.Plan, nameof(message.Plan));
_cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token);
// Features
Features = ApiUtil.GetFeatures(message.Plan);
// Endpoints
Endpoints = message.Environment.Endpoints;
Endpoints.Add(message.Environment.SystemConnection);
// SecureFiles
SecureFiles = message.Environment.SecureFiles;
// Variables (constructor performs initial recursive expansion)
List <string> warnings;
Variables = new Variables(HostContext, message.Environment.Variables, message.Environment.MaskHints, out warnings);
// Prepend Path
PrependPath = new List <string>();
// Docker
Container = new ContainerInfo()
{
ContainerImage = Variables.Get("_PREVIEW_VSTS_DOCKER_IMAGE"),
ContainerName = $"VSTS_{Variables.System_HostType.ToString()}_{message.JobId.ToString("D")}",
};
// Proxy variables
var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress))
{
Variables.Set(Constants.Variables.Agent.ProxyUrl, agentWebProxy.ProxyAddress);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY", string.Empty);
if (!string.IsNullOrEmpty(agentWebProxy.ProxyUsername))
{
Variables.Set(Constants.Variables.Agent.ProxyUsername, agentWebProxy.ProxyUsername);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_USERNAME", string.Empty);
}
if (!string.IsNullOrEmpty(agentWebProxy.ProxyPassword))
{
Variables.Set(Constants.Variables.Agent.ProxyPassword, agentWebProxy.ProxyPassword, true);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_PASSWORD", string.Empty);
}
if (agentWebProxy.ProxyBypassList.Count > 0)
{
Variables.Set(Constants.Variables.Agent.ProxyBypassList, JsonUtility.ToString(agentWebProxy.ProxyBypassList));
}
}
// Job timeline record.
InitializeTimelineRecord(
timelineId: message.Timeline.Id,
timelineRecordId: message.JobId,
parentTimelineRecordId: null,
recordType: ExecutionContextType.Job,
displayName: message.JobName,
refName: message.JobRefName,
order: null); // The job timeline record's order is set by server.
// Logger (must be initialized before writing warnings).
_logger = HostContext.CreateService <IPagingLogger>();
_logger.Setup(_mainTimelineId, _record.Id);
// Log warnings from recursive variable expansion.
warnings?.ForEach(x => this.Warning(x));
// Verbosity (from system.debug).
WriteDebug = Variables.System_Debug ?? false;
// Hook up JobServerQueueThrottling event, we will log warning on server tarpit.
_jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived;
}
private string FormatArgumentsWithDefaults(params string[] args)
{
// Format each arg.
List <string> formattedArgs = new List <string>();
foreach (string arg in args ?? new string[0])
{
if (!string.IsNullOrEmpty(arg))
{
// Validate the arg.
if (arg.IndexOfAny(new char[] { '"', '\r', '\n' }) >= 0)
{
throw new Exception(StringUtil.Loc("InvalidCommandArg", arg));
}
// Add the arg.
formattedArgs.Add(QuotedArgument(arg));
}
}
// Add the common parameters.
if (_acceptUntrusted)
{
formattedArgs.Add("--trust-server-cert");
}
if (!string.IsNullOrWhiteSpace(_username))
{
formattedArgs.Add("--username");
formattedArgs.Add(QuotedArgument(_username));
}
if (!string.IsNullOrWhiteSpace(_password))
{
formattedArgs.Add("--password");
formattedArgs.Add(QuotedArgument(_password));
}
formattedArgs.Add("--no-auth-cache"); // Do not cache credentials
formattedArgs.Add("--non-interactive");
// Add proxy setting parameters
var agentProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(_context.Variables.Agent_ProxyUrl) && !agentProxy.WebProxy.IsBypassed(_repository?.Url ?? _endpoint.Url))
{
_context.Debug($"Add proxy setting parameters to '{_svn}' for proxy server '{_context.Variables.Agent_ProxyUrl}'.");
formattedArgs.Add("--config-option");
formattedArgs.Add(QuotedArgument($"servers:global:http-proxy-host={new Uri(_context.Variables.Agent_ProxyUrl).Host}"));
formattedArgs.Add("--config-option");
formattedArgs.Add(QuotedArgument($"servers:global:http-proxy-port={new Uri(_context.Variables.Agent_ProxyUrl).Port}"));
if (!string.IsNullOrEmpty(_context.Variables.Agent_ProxyUsername))
{
formattedArgs.Add("--config-option");
formattedArgs.Add(QuotedArgument($"servers:global:http-proxy-username={_context.Variables.Agent_ProxyUsername}"));
}
if (!string.IsNullOrEmpty(_context.Variables.Agent_ProxyPassword))
{
formattedArgs.Add("--config-option");
formattedArgs.Add(QuotedArgument($"servers:global:http-proxy-password={_context.Variables.Agent_ProxyPassword}"));
}
}
return(string.Join(" ", formattedArgs));
}
public Definition LoadAction(IExecutionContext executionContext, Pipelines.ActionStep action)
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(action, nameof(action));
// Initialize the definition wrapper object.
var definition = new Definition()
{
Data = new ActionDefinitionData()
};
if (action.Reference.Type == Pipelines.ActionSourceType.ContainerRegistry)
{
Trace.Info("Load action that reference container from registry.");
CachedActionContainers.TryGetValue(action.Id, out var container);
ArgUtil.NotNull(container, nameof(container));
definition.Data.Execution = new ContainerActionExecutionData()
{
Image = container.ContainerImage
};
Trace.Info($"Using action container image: {container.ContainerImage}.");
}
else if (action.Reference.Type == Pipelines.ActionSourceType.Repository)
{
string actionDirectory = null;
var repoAction = action.Reference as Pipelines.RepositoryPathReference;
if (string.Equals(repoAction.RepositoryType, Pipelines.PipelineConstants.SelfAlias, StringComparison.OrdinalIgnoreCase))
{
actionDirectory = executionContext.GetGitHubContext("workspace");
if (!string.IsNullOrEmpty(repoAction.Path))
{
actionDirectory = Path.Combine(actionDirectory, repoAction.Path);
}
}
else
{
actionDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), repoAction.Name.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), repoAction.Ref);
if (!string.IsNullOrEmpty(repoAction.Path))
{
actionDirectory = Path.Combine(actionDirectory, repoAction.Path);
}
}
Trace.Info($"Load action that reference repository from '{actionDirectory}'");
definition.Directory = actionDirectory;
string manifestFile = Path.Combine(actionDirectory, Constants.Path.ActionManifestYmlFile);
string manifestFileYaml = Path.Combine(actionDirectory, Constants.Path.ActionManifestYamlFile);
string dockerFile = Path.Combine(actionDirectory, "Dockerfile");
string dockerFileLowerCase = Path.Combine(actionDirectory, "dockerfile");
if (File.Exists(manifestFile) || File.Exists(manifestFileYaml))
{
var manifestManager = HostContext.GetService <IActionManifestManager>();
if (File.Exists(manifestFile))
{
definition.Data = manifestManager.Load(executionContext, manifestFile);
}
else
{
definition.Data = manifestManager.Load(executionContext, manifestFileYaml);
}
Trace.Verbose($"Action friendly name: '{definition.Data.Name}'");
Trace.Verbose($"Action description: '{definition.Data.Description}'");
if (definition.Data.Inputs != null)
{
foreach (var input in definition.Data.Inputs)
{
Trace.Verbose($"Action input: '{input.Key.ToString()}' default to '{input.Value.ToString()}'");
}
}
if (definition.Data.Execution.ExecutionType == ActionExecutionType.Container)
{
var containerAction = definition.Data.Execution as ContainerActionExecutionData;
Trace.Info($"Action container Dockerfile/image: {containerAction.Image}.");
if (containerAction.Arguments != null)
{
Trace.Info($"Action container args: {StringUtil.ConvertToJson(containerAction.Arguments)}.");
}
if (containerAction.Environment != null)
{
Trace.Info($"Action container env: {StringUtil.ConvertToJson(containerAction.Environment)}.");
}
if (!string.IsNullOrEmpty(containerAction.Pre))
{
Trace.Info($"Action container pre entrypoint: {containerAction.Pre}.");
}
if (!string.IsNullOrEmpty(containerAction.EntryPoint))
{
Trace.Info($"Action container entrypoint: {containerAction.EntryPoint}.");
}
if (!string.IsNullOrEmpty(containerAction.Post))
{
Trace.Info($"Action container post entrypoint: {containerAction.Post}.");
}
if (CachedActionContainers.TryGetValue(action.Id, out var container))
{
Trace.Info($"Image '{containerAction.Image}' already built/pulled, use image: {container.ContainerImage}.");
containerAction.Image = container.ContainerImage;
}
}
else if (definition.Data.Execution.ExecutionType == ActionExecutionType.NodeJS)
{
var nodeAction = definition.Data.Execution as NodeJSActionExecutionData;
Trace.Info($"Action pre node.js file: {nodeAction.Pre ?? "N/A"}.");
Trace.Info($"Action node.js file: {nodeAction.Script}.");
Trace.Info($"Action post node.js file: {nodeAction.Post ?? "N/A"}.");
}
else if (definition.Data.Execution.ExecutionType == ActionExecutionType.Plugin)
{
var pluginAction = definition.Data.Execution as PluginActionExecutionData;
var pluginManager = HostContext.GetService <IRunnerPluginManager>();
var plugin = pluginManager.GetPluginAction(pluginAction.Plugin);
ArgUtil.NotNull(plugin, pluginAction.Plugin);
ArgUtil.NotNullOrEmpty(plugin.PluginTypeName, pluginAction.Plugin);
pluginAction.Plugin = plugin.PluginTypeName;
Trace.Info($"Action plugin: {plugin.PluginTypeName}.");
if (!string.IsNullOrEmpty(plugin.PostPluginTypeName))
{
pluginAction.Post = plugin.PostPluginTypeName;
Trace.Info($"Action cleanup plugin: {plugin.PluginTypeName}.");
}
}
else
{
throw new NotSupportedException(definition.Data.Execution.ExecutionType.ToString());
}
}
else if (File.Exists(dockerFile))
{
if (CachedActionContainers.TryGetValue(action.Id, out var container))
{
definition.Data.Execution = new ContainerActionExecutionData()
{
Image = container.ContainerImage
};
}
else
{
definition.Data.Execution = new ContainerActionExecutionData()
{
Image = dockerFile
};
}
}
else if (File.Exists(dockerFileLowerCase))
{
if (CachedActionContainers.TryGetValue(action.Id, out var container))
{
definition.Data.Execution = new ContainerActionExecutionData()
{
Image = container.ContainerImage
};
}
else
{
definition.Data.Execution = new ContainerActionExecutionData()
{
Image = dockerFileLowerCase
};
}
}
else
{
var fullPath = IOUtil.ResolvePath(actionDirectory, "."); // resolve full path without access filesystem.
throw new NotSupportedException($"Can't find 'action.yml', 'action.yaml' or 'Dockerfile' under '{fullPath}'. Did you forget to run actions/checkout before running your local action?");
}
}
else if (action.Reference.Type == Pipelines.ActionSourceType.Script)
{
definition.Data.Execution = new ScriptActionExecutionData();
definition.Data.Name = "Run";
definition.Data.Description = "Execute a script";
}
else
{
throw new NotSupportedException(action.Reference.Type.ToString());
}
return(definition);
}
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_windowsServiceHelper = HostContext.GetService <INativeWindowsServiceHelper>();
_term = HostContext.GetService <ITerminal>();
}
private ActionContainer PrepareRepositoryActionAsync(IExecutionContext executionContext, Pipelines.ActionStep repositoryAction)
{
var repositoryReference = repositoryAction.Reference as Pipelines.RepositoryPathReference;
if (string.Equals(repositoryReference.RepositoryType, Pipelines.PipelineConstants.SelfAlias, StringComparison.OrdinalIgnoreCase))
{
Trace.Info($"Repository action is in 'self' repository.");
return(null);
}
var setupInfo = new ActionContainer();
string destDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), repositoryReference.Name.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), repositoryReference.Ref);
string actionEntryDirectory = destDirectory;
string dockerFileRelativePath = repositoryReference.Name;
ArgUtil.NotNull(repositoryReference, nameof(repositoryReference));
if (!string.IsNullOrEmpty(repositoryReference.Path))
{
actionEntryDirectory = Path.Combine(destDirectory, repositoryReference.Path);
dockerFileRelativePath = $"{dockerFileRelativePath}/{repositoryReference.Path}";
setupInfo.ActionRepository = $"{repositoryReference.Name}/{repositoryReference.Path}@{repositoryReference.Ref}";
}
else
{
setupInfo.ActionRepository = $"{repositoryReference.Name}@{repositoryReference.Ref}";
}
// find the docker file or action.yml file
var dockerFile = Path.Combine(actionEntryDirectory, "Dockerfile");
var dockerFileLowerCase = Path.Combine(actionEntryDirectory, "dockerfile");
var actionManifest = Path.Combine(actionEntryDirectory, Constants.Path.ActionManifestYmlFile);
var actionManifestYaml = Path.Combine(actionEntryDirectory, Constants.Path.ActionManifestYamlFile);
if (File.Exists(actionManifest) || File.Exists(actionManifestYaml))
{
executionContext.Debug($"action.yml for action: '{actionManifest}'.");
var manifestManager = HostContext.GetService <IActionManifestManager>();
ActionDefinitionData actionDefinitionData = null;
if (File.Exists(actionManifest))
{
actionDefinitionData = manifestManager.Load(executionContext, actionManifest);
}
else
{
actionDefinitionData = manifestManager.Load(executionContext, actionManifestYaml);
}
if (actionDefinitionData.Execution.ExecutionType == ActionExecutionType.Container)
{
var containerAction = actionDefinitionData.Execution as ContainerActionExecutionData;
if (containerAction.Image.EndsWith("Dockerfile") || containerAction.Image.EndsWith("dockerfile"))
{
var dockerFileFullPath = Path.Combine(actionEntryDirectory, containerAction.Image);
executionContext.Debug($"Dockerfile for action: '{dockerFileFullPath}'.");
setupInfo.Dockerfile = dockerFileFullPath;
setupInfo.WorkingDirectory = destDirectory;
return(setupInfo);
}
else if (containerAction.Image.StartsWith("docker://", StringComparison.OrdinalIgnoreCase))
{
var actionImage = containerAction.Image.Substring("docker://".Length);
executionContext.Debug($"Container image for action: '{actionImage}'.");
setupInfo.Image = actionImage;
return(setupInfo);
}
else
{
throw new NotSupportedException($"'{containerAction.Image}' should be either '[path]/Dockerfile' or 'docker://image[:tag]'.");
}
}
else if (actionDefinitionData.Execution.ExecutionType == ActionExecutionType.NodeJS)
{
Trace.Info($"Action node.js file: {(actionDefinitionData.Execution as NodeJSActionExecutionData).Script}, no more preparation.");
return(null);
}
else if (actionDefinitionData.Execution.ExecutionType == ActionExecutionType.Plugin)
{
Trace.Info($"Action plugin: {(actionDefinitionData.Execution as PluginActionExecutionData).Plugin}, no more preparation.");
return(null);
}
else
{
throw new NotSupportedException(actionDefinitionData.Execution.ExecutionType.ToString());
}
}
else if (File.Exists(dockerFile))
{
executionContext.Debug($"Dockerfile for action: '{dockerFile}'.");
setupInfo.Dockerfile = dockerFile;
setupInfo.WorkingDirectory = destDirectory;
return(setupInfo);
}
else if (File.Exists(dockerFileLowerCase))
{
executionContext.Debug($"Dockerfile for action: '{dockerFileLowerCase}'.");
setupInfo.Dockerfile = dockerFileLowerCase;
setupInfo.WorkingDirectory = destDirectory;
return(setupInfo);
}
else
{
var fullPath = IOUtil.ResolvePath(actionEntryDirectory, "."); // resolve full path without access filesystem.
throw new InvalidOperationException($"Can't find 'action.yml', 'action.yaml' or 'Dockerfile' under '{fullPath}'. Did you forget to run actions/checkout before running your local action?");
}
}
public TrackingConfig PrepareDirectory(
IExecutionContext executionContext,
IList <RepositoryResource> repositories,
WorkspaceOptions workspace)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(executionContext.Variables, nameof(executionContext.Variables));
ArgUtil.NotNull(repositories, nameof(repositories));
var trackingManager = HostContext.GetService <ITrackingManager>();
// Create the tracking config for this execution of the pipeline
var agentSettings = HostContext.GetService <IConfigurationStore>().GetSettings();
var newConfig = trackingManager.Create(executionContext, repositories, ShouldOverrideBuildDirectory(repositories, agentSettings));
// Load the tracking config from the last execution of the pipeline
var existingConfig = trackingManager.LoadExistingTrackingConfig(executionContext);
// If there aren't any major changes, merge the configurations and use the same workspace
if (trackingManager.AreTrackingConfigsCompatible(executionContext, newConfig, existingConfig))
{
newConfig = trackingManager.MergeTrackingConfigs(executionContext, newConfig, existingConfig);
}
else if (existingConfig != null)
{
// If the previous config had different repos, get a new workspace folder and mark the old one for clean up
trackingManager.MarkForGarbageCollection(executionContext, existingConfig);
// If the config file was updated to a new config, we need to delete the legacy artifact/staging directories.
// DeleteDirectory will check for the existence of the folders first.
DeleteDirectory(
executionContext,
description: "legacy artifacts directory",
path: Path.Combine(existingConfig.BuildDirectory, Constants.Build.Path.LegacyArtifactsDirectory));
DeleteDirectory(
executionContext,
description: "legacy staging directory",
path: Path.Combine(existingConfig.BuildDirectory, Constants.Build.Path.LegacyStagingDirectory));
}
// Save any changes to the config file
trackingManager.UpdateTrackingConfig(executionContext, newConfig);
// Prepare the build directory.
// There are 2 ways to provide build directory clean policy.
// 1> set definition variable build.clean or agent.clean.buildDirectory. (on-prem user need to use this, since there is no Web UI in TFS 2016)
// 2> select source clean option in definition repository tab. (VSTS will have this option in definition designer UI)
BuildCleanOption cleanOption = GetBuildDirectoryCleanOption(executionContext, workspace);
CreateDirectory(
executionContext,
description: "build directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.BuildDirectory),
deleteExisting: cleanOption == BuildCleanOption.All);
CreateDirectory(
executionContext,
description: "artifacts directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.ArtifactsDirectory),
deleteExisting: true);
CreateDirectory(
executionContext,
description: "test results directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.TestResultsDirectory),
deleteExisting: true);
CreateDirectory(
executionContext,
description: "binaries directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.BuildDirectory, Constants.Build.Path.BinariesDirectory),
deleteExisting: cleanOption == BuildCleanOption.Binary);
CreateDirectory(
executionContext,
description: "source directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.SourcesDirectory),
deleteExisting: cleanOption == BuildCleanOption.Source);
// Set the default clone path for each repository (the Checkout task may override this later)
foreach (var repository in repositories)
{
var repoPath = GetDefaultRepositoryPath(executionContext, repository, newConfig.SourcesDirectory);
Trace.Info($"Set repository path for repository {repository.Alias} to '{repoPath}'");
repository.Properties.Set <string>(RepositoryPropertyNames.Path, repoPath);
}
return(newConfig);
}
public TrackingConfig PrepareDirectory(
IExecutionContext executionContext,
ServiceEndpoint endpoint,
ISourceProvider sourceProvider)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(executionContext.Variables, nameof(executionContext.Variables));
ArgUtil.NotNull(endpoint, nameof(endpoint));
ArgUtil.NotNull(sourceProvider, nameof(sourceProvider));
var trackingManager = HostContext.GetService <ITrackingManager>();
// Defer to the source provider to calculate the hash key.
Trace.Verbose("Calculating build directory hash key.");
string hashKey = sourceProvider.GetBuildDirectoryHashKey(executionContext, endpoint);
Trace.Verbose($"Hash key: {hashKey}");
// Load the existing tracking file if one already exists.
string trackingFile = Path.Combine(
IOUtil.GetWorkPath(HostContext),
Constants.Build.Path.SourceRootMappingDirectory,
executionContext.Variables.System_CollectionId,
executionContext.Variables.System_DefinitionId,
Constants.Build.Path.TrackingConfigFile);
Trace.Verbose($"Loading tracking config if exists: {trackingFile}");
TrackingConfigBase existingConfig = trackingManager.LoadIfExists(executionContext, trackingFile);
// Check if the build needs to be garbage collected. If the hash key
// has changed, then the existing build directory cannot be reused.
TrackingConfigBase garbageConfig = null;
if (existingConfig != null &&
!string.Equals(existingConfig.HashKey, hashKey, StringComparison.OrdinalIgnoreCase))
{
// Just store a reference to the config for now. It can safely be
// marked for garbage collection only after the new build directory
// config has been created.
Trace.Verbose($"Hash key from existing tracking config does not match. Existing key: {existingConfig.HashKey}");
garbageConfig = existingConfig;
existingConfig = null;
}
// Create a new tracking config if required.
TrackingConfig newConfig;
if (existingConfig == null)
{
Trace.Verbose("Creating a new tracking config file.");
newConfig = trackingManager.Create(
executionContext,
endpoint,
hashKey,
trackingFile,
sourceProvider.TestOverrideBuildDirectory());
ArgUtil.NotNull(newConfig, nameof(newConfig));
}
else
{
// Convert legacy format to the new format if required.
newConfig = ConvertToNewFormat(executionContext, endpoint, existingConfig);
// Fill out repository type if it's not there.
// repository type is a new property introduced for maintenance job
if (string.IsNullOrEmpty(newConfig.RepositoryType))
{
newConfig.RepositoryType = endpoint.Type;
}
// For existing tracking config files, update the job run properties.
Trace.Verbose("Updating job run properties.");
trackingManager.UpdateJobRunProperties(executionContext, newConfig, trackingFile);
}
// Mark the old configuration for garbage collection.
if (garbageConfig != null)
{
Trace.Verbose("Marking existing config for garbage collection.");
trackingManager.MarkForGarbageCollection(executionContext, garbageConfig);
}
// Prepare the build directory.
// There are 2 ways to provide build directory clean policy.
// 1> set definition variable build.clean or agent.clean.buildDirectory. (on-prem user need to use this, since there is no Web UI in TFS 2016)
// 2> select source clean option in definition repository tab. (VSTS will have this option in definition designer UI)
BuildCleanOption cleanOption = GetBuildDirectoryCleanOption(executionContext, endpoint);
CreateDirectory(
executionContext,
description: "build directory",
path: Path.Combine(IOUtil.GetWorkPath(HostContext), newConfig.BuildDirectory),
deleteExisting: cleanOption == BuildCleanOption.All);
CreateDirectory(
executionContext,
description: "artifacts directory",
path: Path.Combine(IOUtil.GetWorkPath(HostContext), newConfig.ArtifactsDirectory),
deleteExisting: true);
CreateDirectory(
executionContext,
description: "test results directory",
path: Path.Combine(IOUtil.GetWorkPath(HostContext), newConfig.TestResultsDirectory),
deleteExisting: true);
CreateDirectory(
executionContext,
description: "binaries directory",
path: Path.Combine(IOUtil.GetWorkPath(HostContext), newConfig.BuildDirectory, Constants.Build.Path.BinariesDirectory),
deleteExisting: cleanOption == BuildCleanOption.Binary);
CreateDirectory(
executionContext,
description: "source directory",
path: Path.Combine(IOUtil.GetWorkPath(HostContext), newConfig.BuildDirectory, Constants.Build.Path.SourcesDirectory),
deleteExisting: cleanOption == BuildCleanOption.Source);
return(newConfig);
}
private async Task EnsureDispatchFinished(WorkerDispatcher jobDispatch, bool cancelRunningJob = false)
{
if (!jobDispatch.WorkerDispatch.IsCompleted)
{
if (cancelRunningJob)
{
// cancel running job when shutting down the agent.
// this will happen when agent get Ctrl+C or message queue loop crashed.
jobDispatch.WorkerCancellationTokenSource.Cancel();
// wait for worker process exit then return.
await jobDispatch.WorkerDispatch;
return;
}
// base on the current design, server will only send one job for a given agent everytime.
// if the agent received a new job request while a previous job request is still running, this typically indicate two situations
// 1. an agent bug cause server and agent mismatch on the state of the job request, ex. agent not renew jobrequest properly but think it still own the job reqest, however server already abandon the jobrequest.
// 2. a server bug or design change that allow server send more than one job request to an given agent that haven't finish previous job request.
var agentServer = HostContext.GetService <IAgentServer>();
TaskAgentJobRequest request = null;
try
{
request = await agentServer.GetAgentRequestAsync(_poolId, jobDispatch.RequestId, CancellationToken.None);
}
catch (Exception ex)
{
// we can't even query for the jobrequest from server, something totally busted, stop agent/worker.
Trace.Error($"Catch exception while checking jobrequest {jobDispatch.JobId} status. Cancel running worker right away.");
Trace.Error(ex);
jobDispatch.WorkerCancellationTokenSource.Cancel();
// make sure worker process exit before we rethrow, otherwise we might leave orphan worker process behind.
await jobDispatch.WorkerDispatch;
// rethrow original exception
throw;
}
if (request.Result != null)
{
// job request has been finished, the server already has result.
// this means agent is busted since it still running that request.
// cancel the zombie worker, run next job request.
Trace.Error($"Received job request while previous job {jobDispatch.JobId} still running on worker. Cancel the previous job since the job request have been finished on server side with result: {request.Result.Value}.");
jobDispatch.WorkerCancellationTokenSource.Cancel();
// wait 45 sec for worker to finish.
Task completedTask = await Task.WhenAny(jobDispatch.WorkerDispatch, Task.Delay(TimeSpan.FromSeconds(45)));
if (completedTask != jobDispatch.WorkerDispatch)
{
// at this point, the job exectuion might encounter some dead lock and even not able to be canclled.
// no need to localize the exception string should never happen.
throw new InvalidOperationException($"Job dispatch process for {jobDispatch.JobId} has encountered unexpected error, the dispatch task is not able to be canceled within 45 seconds.");
}
}
else
{
// something seriously wrong on server side. stop agent from continue running.
// no need to localize the exception string should never happen.
throw new InvalidOperationException($"Server send a new job request while the previous job request {jobDispatch.JobId} haven't finished.");
}
}
try
{
await jobDispatch.WorkerDispatch;
Trace.Info($"Job request {jobDispatch.JobId} processed succeed.");
}
catch (Exception ex)
{
Trace.Error($"Worker Dispatch failed witn an exception for job request {jobDispatch.JobId}.");
Trace.Error(ex);
}
finally
{
WorkerDispatcher workerDispatcher;
if (_jobInfos.TryRemove(jobDispatch.JobId, out workerDispatcher))
{
Trace.Verbose($"Remove WorkerDispather from {nameof(_jobInfos)} dictionary for job {jobDispatch.JobId}.");
workerDispatcher.Dispose();
}
}
}
public async Task <TaskAgentMessage> GetNextMessageAsync(CancellationToken token)
{
Trace.Entering();
ArgUtil.NotNull(Session, nameof(Session));
ArgUtil.NotNull(_settings, nameof(_settings));
var agentServer = HostContext.GetService <IAgentServer>();
bool encounteringError = false;
string errorMessage = string.Empty;
while (true)
{
token.ThrowIfCancellationRequested();
TaskAgentMessage message = null;
try
{
message = await agentServer.GetAgentMessageAsync(_settings.PoolId,
Session.SessionId,
_lastMessageId,
token);
// Decrypt the message body if the session is using encryption
message = DecryptMessage(message);
if (message != null)
{
_lastMessageId = message.MessageId;
}
if (encounteringError) //print the message once only if there was an error
{
_term.WriteLine(StringUtil.Loc("QueueConnected", DateTime.UtcNow));
encounteringError = false;
}
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
Trace.Info("Get next message has been cancelled.");
throw;
}
catch (Exception ex)
{
Trace.Error("Catch exception during get next message.");
Trace.Error(ex);
if (ex is TaskAgentSessionExpiredException && await CreateSessionAsync(token))
{
Trace.Info($"{nameof(TaskAgentSessionExpiredException)} received, recoverd by recreate session.");
}
else if (!IsGetNextMessageExceptionRetriable(ex))
{
throw;
}
else
{
//retry after a delay
if (!encounteringError)
{
//print error only on the first consecutive error
_term.WriteError(StringUtil.Loc("QueueConError", DateTime.UtcNow, ex.Message, _getNextMessageRetryInterval.TotalSeconds));
encounteringError = true;
}
Trace.Info("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds);
await HostContext.Delay(_getNextMessageRetryInterval, token);
}
}
if (message == null)
{
Trace.Verbose($"No message retrieved from session '{Session.SessionId}'.");
continue;
}
Trace.Verbose($"Message '{message.MessageId}' received from session '{Session.SessionId}'.");
return(message);
}
}
//create worker manager, create message listener and start listening to the queue
private async Task <int> RunAsync(RunnerSettings settings, bool runOnce = false)
{
try
{
Trace.Info(nameof(RunAsync));
_listener = HostContext.GetService <IMessageListener>();
if (!await _listener.CreateSessionAsync(HostContext.RunnerShutdownToken))
{
return(Constants.Runner.ReturnCode.TerminatedError);
}
HostContext.WritePerfCounter("SessionCreated");
_term.WriteLine($"{DateTime.UtcNow:u}: Listening for Jobs");
IJobDispatcher jobDispatcher = null;
CancellationTokenSource messageQueueLoopTokenSource = CancellationTokenSource.CreateLinkedTokenSource(HostContext.RunnerShutdownToken);
try
{
var notification = HostContext.GetService <IJobNotification>();
notification.StartClient(settings.MonitorSocketAddress);
bool autoUpdateInProgress = false;
Task <bool> selfUpdateTask = null;
bool runOnceJobReceived = false;
jobDispatcher = HostContext.CreateService <IJobDispatcher>();
while (!HostContext.RunnerShutdownToken.IsCancellationRequested)
{
TaskAgentMessage message = null;
bool skipMessageDeletion = false;
try
{
Task <TaskAgentMessage> getNextMessage = _listener.GetNextMessageAsync(messageQueueLoopTokenSource.Token);
if (autoUpdateInProgress)
{
Trace.Verbose("Auto update task running at backend, waiting for getNextMessage or selfUpdateTask to finish.");
Task completeTask = await Task.WhenAny(getNextMessage, selfUpdateTask);
if (completeTask == selfUpdateTask)
{
autoUpdateInProgress = false;
if (await selfUpdateTask)
{
Trace.Info("Auto update task finished at backend, an runner update is ready to apply exit the current runner instance.");
Trace.Info("Stop message queue looping.");
messageQueueLoopTokenSource.Cancel();
try
{
await getNextMessage;
}
catch (Exception ex)
{
Trace.Info($"Ignore any exception after cancel message loop. {ex}");
}
if (runOnce)
{
return(Constants.Runner.ReturnCode.RunOnceRunnerUpdating);
}
else
{
return(Constants.Runner.ReturnCode.RunnerUpdating);
}
}
else
{
Trace.Info("Auto update task finished at backend, there is no available runner update needs to apply, continue message queue looping.");
}
}
}
if (runOnceJobReceived)
{
Trace.Verbose("One time used runner has start running its job, waiting for getNextMessage or the job to finish.");
Task completeTask = await Task.WhenAny(getNextMessage, jobDispatcher.RunOnceJobCompleted.Task);
if (completeTask == jobDispatcher.RunOnceJobCompleted.Task)
{
Trace.Info("Job has finished at backend, the runner will exit since it is running under onetime use mode.");
Trace.Info("Stop message queue looping.");
messageQueueLoopTokenSource.Cancel();
try
{
await getNextMessage;
}
catch (Exception ex)
{
Trace.Info($"Ignore any exception after cancel message loop. {ex}");
}
return(Constants.Runner.ReturnCode.Success);
}
}
message = await getNextMessage; //get next message
HostContext.WritePerfCounter($"MessageReceived_{message.MessageType}");
if (string.Equals(message.MessageType, AgentRefreshMessage.MessageType, StringComparison.OrdinalIgnoreCase))
{
if (autoUpdateInProgress == false)
{
autoUpdateInProgress = true;
var runnerUpdateMessage = JsonUtility.FromString <AgentRefreshMessage>(message.Body);
var selfUpdater = HostContext.GetService <ISelfUpdater>();
selfUpdateTask = selfUpdater.SelfUpdate(runnerUpdateMessage, jobDispatcher, !runOnce && HostContext.StartupType != StartupType.Service, HostContext.RunnerShutdownToken);
Trace.Info("Refresh message received, kick-off selfupdate background process.");
}
else
{
Trace.Info("Refresh message received, skip autoupdate since a previous autoupdate is already running.");
}
}
else if (string.Equals(message.MessageType, JobRequestMessageTypes.PipelineAgentJobRequest, StringComparison.OrdinalIgnoreCase))
{
if (autoUpdateInProgress || runOnceJobReceived)
{
skipMessageDeletion = true;
Trace.Info($"Skip message deletion for job request message '{message.MessageId}'.");
}
else
{
var jobMessage = StringUtil.ConvertFromJson <Pipelines.AgentJobRequestMessage>(message.Body);
jobDispatcher.Run(jobMessage, runOnce);
if (runOnce)
{
Trace.Info("One time used runner received job message.");
runOnceJobReceived = true;
}
}
}
else if (string.Equals(message.MessageType, JobCancelMessage.MessageType, StringComparison.OrdinalIgnoreCase))
{
var cancelJobMessage = JsonUtility.FromString <JobCancelMessage>(message.Body);
bool jobCancelled = jobDispatcher.Cancel(cancelJobMessage);
skipMessageDeletion = (autoUpdateInProgress || runOnceJobReceived) && !jobCancelled;
if (skipMessageDeletion)
{
Trace.Info($"Skip message deletion for cancellation message '{message.MessageId}'.");
}
}
else
{
Trace.Error($"Received message {message.MessageId} with unsupported message type {message.MessageType}.");
}
}
finally
{
if (!skipMessageDeletion && message != null)
{
try
{
await _listener.DeleteMessageAsync(message);
}
catch (Exception ex)
{
Trace.Error($"Catch exception during delete message from message queue. message id: {message.MessageId}");
Trace.Error(ex);
}
finally
{
message = null;
}
}
}
}
}
finally
{
if (jobDispatcher != null)
{
await jobDispatcher.ShutdownAsync();
}
//TODO: make sure we don't mask more important exception
await _listener.DeleteSessionAsync();
messageQueueLoopTokenSource.Dispose();
}
}
catch (TaskAgentAccessTokenExpiredException)
{
Trace.Info("Runner OAuth token has been revoked. Shutting down.");
}
return(Constants.Runner.ReturnCode.Success);
}