public async Task RunAsync() { // Validate args. Trace.Entering(); ArgUtil.NotNull(Data, nameof(Data)); ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext)); ArgUtil.NotNull(Inputs, nameof(Inputs)); ArgUtil.NotNull(TaskDirectory, nameof(TaskDirectory)); // Update the env dictionary. AddInputsToEnvironment(); AddEndpointsToEnvironment(); AddVariablesToEnvironment(); // Resolve the target script. string target = Data.Target; ArgUtil.NotNullOrEmpty(target, nameof(target)); target = Path.Combine(TaskDirectory, target); if (!File.Exists(target)) { throw new Exception(StringUtil.Loc("TaskScriptDoesNotExist", target)); } // Resolve the working directory. string workingDirectory = Data.WorkingDirectory; if (string.IsNullOrEmpty(workingDirectory)) { workingDirectory = TaskDirectory; } else if (!Directory.Exists(workingDirectory)) { throw new Exception(StringUtil.Loc("TaskWorkingDirectoryDoesNotExist", workingDirectory)); } // Setup the process invoker. using (var processInvoker = HostContext.CreateService <IProcessInvoker>()) { object outputLock = new object(); processInvoker.OutputDataReceived += OnDataReceived; processInvoker.ErrorDataReceived += OnDataReceived; string node = Path.Combine( IOUtil.GetExternalsPath(), "node", "bin", $"node{IOUtil.ExeExtension}"); // Format the arguments passed to node. // 1) Wrap the script file path in double quotes. // 2) Escape double quotes within the script file path. Double-quote is a valid // file name character on Linux. string arguments = StringUtil.Format(@"""{0}""", target.Replace(@"""", @"\""")); // Execute the process. Exit code 0 should always be returned. // A non-zero exit code indicates infrastructural failure. // Task failure should be communicated over STDOUT using ## commands. await processInvoker.ExecuteAsync( workingDirectory : workingDirectory, fileName : node, arguments : arguments, environment : Environment, requireExitCodeZero : true, cancellationToken : ExecutionContext.CancellationToken); } }
private async Task <CheckResult> CheckGit(string url, string pat, bool extraCA = false) { var result = new CheckResult(); try { result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************"); result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****"); result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** Validate server cert and proxy configuration with Git "); result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****"); result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************"); var repoUrlBuilder = new UriBuilder(url); repoUrlBuilder.Path = "actions/checkout"; repoUrlBuilder.UserName = "******"; repoUrlBuilder.Password = pat; var gitProxy = ""; var proxy = HostContext.WebProxy.GetProxy(repoUrlBuilder.Uri); if (proxy != null) { result.Logs.Add($"{DateTime.UtcNow.ToString("O")} Runner is behind http proxy '{proxy.AbsoluteUri}'"); if (HostContext.WebProxy.HttpProxyUsername != null || HostContext.WebProxy.HttpsProxyUsername != null) { var proxyUrlWithCred = UrlUtil.GetCredentialEmbeddedUrl( proxy, HostContext.WebProxy.HttpProxyUsername ?? HostContext.WebProxy.HttpsProxyUsername, HostContext.WebProxy.HttpProxyPassword ?? HostContext.WebProxy.HttpsProxyPassword); gitProxy = $"-c http.proxy={proxyUrlWithCred}"; } else { gitProxy = $"-c http.proxy={proxy.AbsoluteUri}"; } } using (var processInvoker = HostContext.CreateService <IProcessInvoker>()) { processInvoker.OutputDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) => { if (!string.IsNullOrEmpty(args.Data)) { result.Logs.Add($"{DateTime.UtcNow.ToString("O")} {args.Data}"); } }); processInvoker.ErrorDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) => { if (!string.IsNullOrEmpty(args.Data)) { result.Logs.Add($"{DateTime.UtcNow.ToString("O")} {args.Data}"); } }); var gitArgs = $"{gitProxy} ls-remote --exit-code {repoUrlBuilder.Uri.AbsoluteUri} HEAD"; result.Logs.Add($"{DateTime.UtcNow.ToString("O")} Run 'git {gitArgs}' "); var env = new Dictionary <string, string> { { "GIT_TRACE", "1" }, { "GIT_CURL_VERBOSE", "1" } }; if (extraCA) { env["GIT_SSL_CAINFO"] = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), "download_ca_cert.pem"); } await processInvoker.ExecuteAsync( HostContext.GetDirectory(WellKnownDirectory.Root), _gitPath, gitArgs, env, true, CancellationToken.None); } result.Pass = true; } catch (Exception ex) { result.Pass = false; result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************"); result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****"); result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** git ls-remote failed with error: {ex}"); if (result.Logs.Any(x => x.Contains("SSL Certificate problem", StringComparison.OrdinalIgnoreCase))) { result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** git ls-remote failed due to SSL cert issue."); result.SslError = true; } result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****"); result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************"); } return(result); }
public async Task <int> RunAsync(string pipeIn, string pipeOut) { // Validate args. ArgUtil.NotNullOrEmpty(pipeIn, nameof(pipeIn)); ArgUtil.NotNullOrEmpty(pipeOut, nameof(pipeOut)); var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>(); var agentCertManager = HostContext.GetService <IAgentCertificateManager>(); ApiUtil.InitializeVssClientSettings(agentWebProxy, agentCertManager); var jobRunner = HostContext.CreateService <IJobRunner>(); using (var channel = HostContext.CreateService <IProcessChannel>()) using (var jobRequestCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(HostContext.AgentShutdownToken)) using (var channelTokenSource = new CancellationTokenSource()) { // Start the channel. channel.StartClient(pipeIn, pipeOut); // Wait for up to 30 seconds for a message from the channel. Trace.Info("Waiting to receive the job message from the channel."); WorkerMessage channelMessage; using (var csChannelMessage = new CancellationTokenSource(_workerStartTimeout)) { channelMessage = await channel.ReceiveAsync(csChannelMessage.Token); } // Deserialize the job message. Trace.Info("Message received."); ArgUtil.Equal(MessageType.NewJobRequest, channelMessage.MessageType, nameof(channelMessage.MessageType)); ArgUtil.NotNullOrEmpty(channelMessage.Body, nameof(channelMessage.Body)); var jobMessage = JsonUtility.FromString <Pipelines.AgentJobRequestMessage>(channelMessage.Body); ArgUtil.NotNull(jobMessage, nameof(jobMessage)); // Initialize the secret masker and set the thread culture. InitializeSecretMasker(jobMessage); SetCulture(jobMessage); // Start the job. Trace.Info($"Job message:{Environment.NewLine} {StringUtil.ConvertToJson(jobMessage)}"); Task <TaskResult> jobRunnerTask = jobRunner.RunAsync(jobMessage, jobRequestCancellationToken.Token); // Start listening for a cancel message from the channel. Trace.Info("Listening for cancel message from the channel."); Task <WorkerMessage> channelTask = channel.ReceiveAsync(channelTokenSource.Token); // Wait for one of the tasks to complete. Trace.Info("Waiting for the job to complete or for a cancel message from the channel."); Task.WaitAny(jobRunnerTask, channelTask); // Handle if the job completed. if (jobRunnerTask.IsCompleted) { Trace.Info("Job completed."); channelTokenSource.Cancel(); // Cancel waiting for a message from the channel. return(TaskResultUtil.TranslateToReturnCode(await jobRunnerTask)); } // Otherwise a cancel message was received from the channel. Trace.Info("Cancellation/Shutdown message received."); channelMessage = await channelTask; switch (channelMessage.MessageType) { case MessageType.CancelRequest: jobRequestCancellationToken.Cancel(); // Expire the host cancellation token. break; case MessageType.AgentShutdown: HostContext.ShutdownAgent(ShutdownReason.UserCancelled); break; case MessageType.OperatingSystemShutdown: HostContext.ShutdownAgent(ShutdownReason.OperatingSystemShutdown); break; default: throw new ArgumentOutOfRangeException(nameof(channelMessage.MessageType), channelMessage.MessageType, nameof(channelMessage.MessageType)); } // Await the job. return(TaskResultUtil.TranslateToReturnCode(await jobRunnerTask)); } }
public async Task <TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken) { // Validate parameters. Trace.Entering(); ArgUtil.NotNull(message, nameof(message)); ArgUtil.NotNull(message.Resources, nameof(message.Resources)); ArgUtil.NotNull(message.Variables, nameof(message.Variables)); ArgUtil.NotNull(message.Steps, nameof(message.Steps)); Trace.Info("Job ID {0}", message.JobId); DateTime jobStartTimeUtc = DateTime.UtcNow; ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); // System.AccessToken if (message.Variables.ContainsKey(Constants.Variables.System.EnableAccessToken) && StringUtil.ConvertToBoolean(message.Variables[Constants.Variables.System.EnableAccessToken].Value)) { message.Variables[Constants.Variables.System.AccessToken] = new VariableValue(systemConnection.Authorization.Parameters["AccessToken"], false); } // back compat TfsServerUrl message.Variables[Constants.Variables.System.TFServerUrl] = systemConnection.Url.AbsoluteUri; // Make sure SystemConnection Url and Endpoint Url match Config Url base for OnPremises server // System.ServerType will always be there after M133 if (!message.Variables.ContainsKey(Constants.Variables.System.ServerType) || string.Equals(message.Variables[Constants.Variables.System.ServerType]?.Value, "OnPremises", StringComparison.OrdinalIgnoreCase)) { ReplaceConfigUriBaseInJobRequestMessage(message); } // Setup the job server and job server queue. var jobServer = HostContext.GetService <IJobServer>(); VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection); Uri jobServerUrl = systemConnection.Url; Trace.Info($"Creating job server with URL: {jobServerUrl}"); // jobServerQueue is the throttling reporter. _jobServerQueue = HostContext.GetService <IJobServerQueue>(); VssConnection jobConnection = VssUtil.CreateConnection(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(_jobServerQueue) }); await jobServer.ConnectAsync(jobConnection); _jobServerQueue.Start(message); HostContext.WritePerfCounter($"WorkerJobServerQueueStarted_{message.RequestId.ToString()}"); IExecutionContext jobContext = null; CancellationTokenRegistration?agentShutdownRegistration = null; try { // Create the job execution context. jobContext = HostContext.CreateService <IExecutionContext>(); jobContext.InitializeJob(message, jobRequestCancellationToken); Trace.Info("Starting the job execution context."); jobContext.Start(); jobContext.Section(StringUtil.Loc("StepStarting", message.JobDisplayName)); agentShutdownRegistration = HostContext.AgentShutdownToken.Register(() => { // log an issue, then agent get shutdown by Ctrl-C or Ctrl-Break. // the server will use Ctrl-Break to tells the agent that operating system is shutting down. string errorMessage; switch (HostContext.AgentShutdownReason) { case ShutdownReason.UserCancelled: errorMessage = StringUtil.Loc("UserShutdownAgent"); break; case ShutdownReason.OperatingSystemShutdown: errorMessage = StringUtil.Loc("OperatingSystemShutdown", Environment.MachineName); break; default: throw new ArgumentException(HostContext.AgentShutdownReason.ToString(), nameof(HostContext.AgentShutdownReason)); } jobContext.AddIssue(new Issue() { Type = IssueType.Error, Message = errorMessage }); }); // Validate directory permissions. string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work); Trace.Info($"Validating directory permissions for: '{workDirectory}'"); try { Directory.CreateDirectory(workDirectory); IOUtil.ValidateExecutePermission(workDirectory); } catch (Exception ex) { Trace.Error(ex); jobContext.Error(ex); return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed)); } // Set agent variables. AgentSettings settings = HostContext.GetService <IConfigurationStore>().GetSettings(); jobContext.SetVariable(Constants.Variables.Agent.Id, settings.AgentId.ToString(CultureInfo.InvariantCulture)); jobContext.SetVariable(Constants.Variables.Agent.HomeDirectory, HostContext.GetDirectory(WellKnownDirectory.Root), isFilePath: true); jobContext.SetVariable(Constants.Variables.Agent.JobName, message.JobDisplayName); jobContext.SetVariable(Constants.Variables.Agent.MachineName, Environment.MachineName); jobContext.SetVariable(Constants.Variables.Agent.Name, settings.AgentName); jobContext.SetVariable(Constants.Variables.Agent.OS, VarUtil.OS); jobContext.SetVariable(Constants.Variables.Agent.OSArchitecture, VarUtil.OSArchitecture); jobContext.SetVariable(Constants.Variables.Agent.RootDirectory, HostContext.GetDirectory(WellKnownDirectory.Work), isFilePath: true); if (PlatformUtil.RunningOnWindows) { jobContext.SetVariable(Constants.Variables.Agent.ServerOMDirectory, HostContext.GetDirectory(WellKnownDirectory.ServerOM), isFilePath: true); } if (!PlatformUtil.RunningOnWindows) { jobContext.SetVariable(Constants.Variables.Agent.AcceptTeeEula, settings.AcceptTeeEula.ToString()); } jobContext.SetVariable(Constants.Variables.Agent.WorkFolder, HostContext.GetDirectory(WellKnownDirectory.Work), isFilePath: true); jobContext.SetVariable(Constants.Variables.System.WorkFolder, HostContext.GetDirectory(WellKnownDirectory.Work), isFilePath: true); string toolsDirectory = HostContext.GetDirectory(WellKnownDirectory.Tools); Directory.CreateDirectory(toolsDirectory); jobContext.SetVariable(Constants.Variables.Agent.ToolsDirectory, toolsDirectory, isFilePath: true); if (AgentKnobs.DisableGitPrompt.GetValue(jobContext).AsBoolean()) { jobContext.SetVariable("GIT_TERMINAL_PROMPT", "0"); } // Setup TEMP directories _tempDirectoryManager = HostContext.GetService <ITempDirectoryManager>(); _tempDirectoryManager.InitializeTempDirectory(jobContext); // todo: task server can throw. try/catch and fail job gracefully. // prefer task definitions url, then TFS collection url, then TFS account url var taskServer = HostContext.GetService <ITaskServer>(); Uri taskServerUri = null; if (!string.IsNullOrEmpty(jobContext.Variables.System_TaskDefinitionsUri)) { taskServerUri = new Uri(jobContext.Variables.System_TaskDefinitionsUri); } else if (!string.IsNullOrEmpty(jobContext.Variables.System_TFCollectionUrl)) { taskServerUri = new Uri(jobContext.Variables.System_TFCollectionUrl); } var taskServerCredential = VssUtil.GetVssCredential(systemConnection); if (taskServerUri != null) { Trace.Info($"Creating task server with {taskServerUri}"); await taskServer.ConnectAsync(VssUtil.CreateConnection(taskServerUri, taskServerCredential)); } // for back compat TFS 2015 RTM/QU1, we may need to switch the task server url to agent config url if (!string.Equals(message?.Variables.GetValueOrDefault(Constants.Variables.System.ServerType)?.Value, "Hosted", StringComparison.OrdinalIgnoreCase)) { if (taskServerUri == null || !await taskServer.TaskDefinitionEndpointExist()) { Trace.Info($"Can't determine task download url from JobMessage or the endpoint doesn't exist."); var configStore = HostContext.GetService <IConfigurationStore>(); taskServerUri = new Uri(configStore.GetSettings().ServerUrl); Trace.Info($"Recreate task server with configuration server url: {taskServerUri}"); await taskServer.ConnectAsync(VssUtil.CreateConnection(taskServerUri, taskServerCredential)); } } // Expand the endpoint data values. foreach (ServiceEndpoint endpoint in jobContext.Endpoints) { jobContext.Variables.ExpandValues(target: endpoint.Data); VarUtil.ExpandEnvironmentVariables(HostContext, target: endpoint.Data); } // Expand the repository property values. foreach (var repository in jobContext.Repositories) { // expand checkout option var checkoutOptions = repository.Properties.Get <JToken>(Pipelines.RepositoryPropertyNames.CheckoutOptions); if (checkoutOptions != null) { checkoutOptions = jobContext.Variables.ExpandValues(target: checkoutOptions); checkoutOptions = VarUtil.ExpandEnvironmentVariables(HostContext, target: checkoutOptions); repository.Properties.Set <JToken>(Pipelines.RepositoryPropertyNames.CheckoutOptions, checkoutOptions);; } // expand workspace mapping var mappings = repository.Properties.Get <JToken>(Pipelines.RepositoryPropertyNames.Mappings); if (mappings != null) { mappings = jobContext.Variables.ExpandValues(target: mappings); mappings = VarUtil.ExpandEnvironmentVariables(HostContext, target: mappings); repository.Properties.Set <JToken>(Pipelines.RepositoryPropertyNames.Mappings, mappings); } } // Expand container properties foreach (var container in jobContext.Containers) { this.ExpandProperties(container, jobContext.Variables); } foreach (var sidecar in jobContext.SidecarContainers) { this.ExpandProperties(sidecar, jobContext.Variables); } // Get the job extension. Trace.Info("Getting job extension."); var hostType = jobContext.Variables.System_HostType; var extensionManager = HostContext.GetService <IExtensionManager>(); // We should always have one job extension IJobExtension jobExtension = (extensionManager.GetExtensions <IJobExtension>() ?? new List <IJobExtension>()) .Where(x => x.HostType.HasFlag(hostType)) .FirstOrDefault(); ArgUtil.NotNull(jobExtension, nameof(jobExtension)); List <IStep> jobSteps = null; try { Trace.Info("Initialize job. Getting all job steps."); jobSteps = await jobExtension.InitializeJob(jobContext, message); } catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested) { // set the job to canceled // don't log error issue to job ExecutionContext, since server owns the job level issue Trace.Error($"Job is canceled during initialize."); Trace.Error($"Caught exception: {ex}"); return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled)); } catch (Exception ex) { // set the job to failed. // don't log error issue to job ExecutionContext, since server owns the job level issue Trace.Error($"Job initialize failed."); Trace.Error($"Caught exception from {nameof(jobExtension.InitializeJob)}: {ex}"); return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed)); } // trace out all steps Trace.Info($"Total job steps: {jobSteps.Count}."); Trace.Verbose($"Job steps: '{string.Join(", ", jobSteps.Select(x => x.DisplayName))}'"); HostContext.WritePerfCounter($"WorkerJobInitialized_{message?.RequestId.ToString()}"); // Run all job steps Trace.Info("Run all job steps."); var stepsRunner = HostContext.GetService <IStepsRunner>(); try { await stepsRunner.RunAsync(jobContext, jobSteps); } catch (Exception ex) { // StepRunner should never throw exception out. // End up here mean there is a bug in StepRunner // Log the error and fail the job. Trace.Error($"Caught exception from job steps {nameof(StepsRunner)}: {ex}"); jobContext.Error(ex); return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed)); } finally { Trace.Info("Finalize job."); await jobExtension.FinalizeJob(jobContext); } Trace.Info($"Job result after all job steps finish: {jobContext.Result ?? TaskResult.Succeeded}"); if (jobContext.Variables.GetBoolean(Constants.Variables.Agent.Diagnostic) ?? false) { Trace.Info("Support log upload starting."); IDiagnosticLogManager diagnosticLogManager = HostContext.GetService <IDiagnosticLogManager>(); try { await diagnosticLogManager.UploadDiagnosticLogsAsync(executionContext : jobContext, message : message, jobStartTimeUtc : jobStartTimeUtc); Trace.Info("Support log upload complete."); } catch (Exception ex) { // Log the error but make sure we continue gracefully. Trace.Info("Error uploading support logs."); Trace.Error(ex); } } Trace.Info("Completing the job execution context."); return(await CompleteJobAsync(jobServer, jobContext, message)); } finally { if (agentShutdownRegistration != null) { agentShutdownRegistration.Value.Dispose(); agentShutdownRegistration = null; } await ShutdownQueue(throwOnFailure : false); } }
// download all required tasks. // make sure all task's condition inputs are valid. // build up three list of steps for jobrunner. (pre-job, job, post-job) public async Task <JobInitializeResult> InitializeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message) { Trace.Entering(); ArgUtil.NotNull(jobContext, nameof(jobContext)); ArgUtil.NotNull(message, nameof(message)); // create a new timeline record node for 'Initialize job' IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("InitializeJob"), nameof(JobExtension)); JobInitializeResult initResult = new JobInitializeResult(); using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); })) { try { context.Start(); context.Section(StringUtil.Loc("StepStarting", StringUtil.Loc("InitializeJob"))); // Set agent version variable. context.Variables.Set(Constants.Variables.Agent.Version, Constants.Agent.Version); context.Output(StringUtil.Loc("AgentVersion", Constants.Agent.Version)); // Print proxy setting information for better diagnostic experience var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>(); if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress)) { context.Output(StringUtil.Loc("AgentRunningBehindProxy", agentWebProxy.ProxyAddress)); } // Give job extension a chance to initialize Trace.Info($"Run initial step from extension {this.GetType().Name}."); InitializeJobExtension(context, message.Steps, message.Workspace); // Download tasks if not already in the cache Trace.Info("Downloading task definitions."); var taskManager = HostContext.GetService <ITaskManager>(); await taskManager.DownloadAsync(context, message.Steps); // Parse all Task conditions. Trace.Info("Parsing all task's condition inputs."); var expression = HostContext.GetService <IExpressionManager>(); Dictionary <Guid, IExpressionNode> taskConditionMap = new Dictionary <Guid, IExpressionNode>(); foreach (var task in message.Steps.OfType <Pipelines.TaskStep>()) { IExpressionNode condition; if (!string.IsNullOrEmpty(task.Condition)) { context.Debug($"Task '{task.DisplayName}' has following condition: '{task.Condition}'."); condition = expression.Parse(context, task.Condition); } else { condition = ExpressionManager.Succeeded; } taskConditionMap[task.Id] = condition; } #if OS_WINDOWS // This is for internal testing and is not publicly supported. This will be removed from the agent at a later time. var prepareScript = Environment.GetEnvironmentVariable("VSTS_AGENT_INIT_INTERNAL_TEMP_HACK"); ServiceEndpoint systemConnection = context.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); if (!string.IsNullOrEmpty(prepareScript) && context.Container == null) { var prepareStep = new ManagementScriptStep( scriptPath: prepareScript, condition: ExpressionManager.Succeeded, displayName: "Agent Initialization"); Trace.Verbose($"Adding agent init script step."); prepareStep.Initialize(HostContext); prepareStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), prepareStep.DisplayName, nameof(ManagementScriptStep)); prepareStep.AccessToken = systemConnection.Authorization.Parameters["AccessToken"]; prepareStep.Condition = ExpressionManager.Succeeded; initResult.PreJobSteps.Add(prepareStep); } #endif // build up 3 lists of steps, pre-job, job, post-job Stack <IStep> postJobStepsBuilder = new Stack <IStep>(); Dictionary <Guid, Variables> taskVariablesMapping = new Dictionary <Guid, Variables>(); if (context.Container != null) { var containerProvider = HostContext.GetService <IContainerOperationProvider>(); initResult.PreJobSteps.Add(new JobExtensionRunner(runAsync: containerProvider.StartContainerAsync, condition: ExpressionManager.Succeeded, displayName: StringUtil.Loc("InitializeContainer"), data: (object)jobContext.Container)); postJobStepsBuilder.Push(new JobExtensionRunner(runAsync: containerProvider.StopContainerAsync, condition: ExpressionManager.Always, displayName: StringUtil.Loc("StopContainer"), data: (object)jobContext.Container)); } foreach (var task in message.Steps.OfType <Pipelines.TaskStep>()) { var taskDefinition = taskManager.Load(task); List <string> warnings; taskVariablesMapping[task.Id] = new Variables(HostContext, new Dictionary <string, VariableValue>(), out warnings); // Add pre-job steps from Tasks if (taskDefinition.Data?.PreJobExecution != null) { Trace.Info($"Adding Pre-Job {task.DisplayName}."); var taskRunner = HostContext.CreateService <ITaskRunner>(); taskRunner.Task = task; taskRunner.Stage = JobRunStage.PreJob; taskRunner.Condition = taskConditionMap[task.Id]; initResult.PreJobSteps.Add(taskRunner); } // Add execution steps from Tasks if (taskDefinition.Data?.Execution != null) { Trace.Verbose($"Adding {task.DisplayName}."); var taskRunner = HostContext.CreateService <ITaskRunner>(); taskRunner.Task = task; taskRunner.Stage = JobRunStage.Main; taskRunner.Condition = taskConditionMap[task.Id]; initResult.JobSteps.Add(taskRunner); } // Add post-job steps from Tasks if (taskDefinition.Data?.PostJobExecution != null) { Trace.Verbose($"Adding Post-Job {task.DisplayName}."); var taskRunner = HostContext.CreateService <ITaskRunner>(); taskRunner.Task = task; taskRunner.Stage = JobRunStage.PostJob; taskRunner.Condition = ExpressionManager.Always; postJobStepsBuilder.Push(taskRunner); } } // Add pre-job step from Extension Trace.Info("Adding pre-job step from extension."); var extensionPreJobStep = GetExtensionPreJobStep(jobContext); if (extensionPreJobStep != null) { initResult.PreJobSteps.Add(extensionPreJobStep); } // Add post-job step from Extension Trace.Info("Adding post-job step from extension."); var extensionPostJobStep = GetExtensionPostJobStep(jobContext); if (extensionPostJobStep != null) { postJobStepsBuilder.Push(extensionPostJobStep); } // create execution context for all pre-job steps foreach (var step in initResult.PreJobSteps) { #if OS_WINDOWS if (step is ManagementScriptStep) { continue; } #endif if (step is JobExtensionRunner) { JobExtensionRunner extensionStep = step as JobExtensionRunner; ArgUtil.NotNull(extensionStep, extensionStep.DisplayName); Guid stepId = Guid.NewGuid(); extensionStep.ExecutionContext = jobContext.CreateChild(stepId, extensionStep.DisplayName, stepId.ToString("N")); } else if (step is ITaskRunner) { ITaskRunner taskStep = step as ITaskRunner; ArgUtil.NotNull(taskStep, step.DisplayName); taskStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("PreJob", taskStep.DisplayName), taskStep.Task.Name, taskVariablesMapping[taskStep.Task.Id]); } } // create task execution context for all job steps from task foreach (var step in initResult.JobSteps) { ITaskRunner taskStep = step as ITaskRunner; ArgUtil.NotNull(taskStep, step.DisplayName); taskStep.ExecutionContext = jobContext.CreateChild(taskStep.Task.Id, taskStep.DisplayName, taskStep.Task.Name, taskVariablesMapping[taskStep.Task.Id]); } // Add post-job steps from Tasks Trace.Info("Adding post-job steps from tasks."); while (postJobStepsBuilder.Count > 0) { initResult.PostJobStep.Add(postJobStepsBuilder.Pop()); } // create task execution context for all post-job steps from task foreach (var step in initResult.PostJobStep) { if (step is JobExtensionRunner) { JobExtensionRunner extensionStep = step as JobExtensionRunner; ArgUtil.NotNull(extensionStep, extensionStep.DisplayName); Guid stepId = Guid.NewGuid(); extensionStep.ExecutionContext = jobContext.CreateChild(stepId, extensionStep.DisplayName, stepId.ToString("N")); } else if (step is ITaskRunner) { ITaskRunner taskStep = step as ITaskRunner; ArgUtil.NotNull(taskStep, step.DisplayName); taskStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("PostJob", taskStep.DisplayName), taskStep.Task.Name, taskVariablesMapping[taskStep.Task.Id]); } } #if OS_WINDOWS // Add script post steps. // This is for internal testing and is not publicly supported. This will be removed from the agent at a later time. var finallyScript = Environment.GetEnvironmentVariable("VSTS_AGENT_CLEANUP_INTERNAL_TEMP_HACK"); if (!string.IsNullOrEmpty(finallyScript) && context.Container == null) { var finallyStep = new ManagementScriptStep( scriptPath: finallyScript, condition: ExpressionManager.Always, displayName: "Agent Cleanup"); Trace.Verbose($"Adding agent cleanup script step."); finallyStep.Initialize(HostContext); finallyStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), finallyStep.DisplayName, nameof(ManagementScriptStep)); finallyStep.Condition = ExpressionManager.Always; finallyStep.AccessToken = systemConnection.Authorization.Parameters["AccessToken"]; initResult.PostJobStep.Add(finallyStep); } #endif return(initResult); } catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested) { // Log the exception and cancel the JobExtension Initialization. Trace.Error($"Caught cancellation exception from JobExtension Initialization: {ex}"); context.Error(ex); context.Result = TaskResult.Canceled; throw; } catch (Exception ex) { // Log the error and fail the JobExtension Initialization. Trace.Error($"Caught exception from JobExtension Initialization: {ex}"); context.Error(ex); context.Result = TaskResult.Failed; throw; } finally { context.Section(StringUtil.Loc("StepFinishing", StringUtil.Loc("InitializeJob"))); context.Complete(); } } }
public IHandler Create( IExecutionContext executionContext, Pipelines.TaskStepDefinitionReference task, IStepHost stepHost, List <ServiceEndpoint> endpoints, List <SecureFile> secureFiles, HandlerData data, Dictionary <string, string> inputs, Dictionary <string, string> environment, string taskDirectory, string filePathInputRootDirectory) { // Validate args. Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(stepHost, nameof(stepHost)); ArgUtil.NotNull(endpoints, nameof(endpoints)); ArgUtil.NotNull(secureFiles, nameof(secureFiles)); ArgUtil.NotNull(data, nameof(data)); ArgUtil.NotNull(inputs, nameof(inputs)); ArgUtil.NotNull(environment, nameof(environment)); ArgUtil.NotNull(taskDirectory, nameof(taskDirectory)); // Create the handler. IHandler handler; if (data is NodeHandlerData) { // Node. handler = HostContext.CreateService <INodeHandler>(); (handler as INodeHandler).Data = data as NodeHandlerData; } else if (data is PowerShell3HandlerData) { // PowerShell3. handler = HostContext.CreateService <IPowerShell3Handler>(); (handler as IPowerShell3Handler).Data = data as PowerShell3HandlerData; } else if (data is PowerShellExeHandlerData) { // PowerShellExe. handler = HostContext.CreateService <IPowerShellExeHandler>(); (handler as IPowerShellExeHandler).Data = data as PowerShellExeHandlerData; } else if (data is ProcessHandlerData) { // Process. handler = HostContext.CreateService <IProcessHandler>(); (handler as IProcessHandler).Data = data as ProcessHandlerData; } else if (data is PowerShellHandlerData) { // PowerShell. handler = HostContext.CreateService <IPowerShellHandler>(); (handler as IPowerShellHandler).Data = data as PowerShellHandlerData; } else if (data is AzurePowerShellHandlerData) { // AzurePowerShell. handler = HostContext.CreateService <IAzurePowerShellHandler>(); (handler as IAzurePowerShellHandler).Data = data as AzurePowerShellHandlerData; } else if (data is AgentPluginHandlerData) { // Agent plugin handler = HostContext.CreateService <IAgentPluginHandler>(); (handler as IAgentPluginHandler).Data = data as AgentPluginHandlerData; } else { // This should never happen. throw new NotSupportedException(); } handler.Endpoints = endpoints; handler.Task = task; handler.Environment = environment; handler.ExecutionContext = executionContext; handler.StepHost = stepHost; handler.FilePathInputRootDirectory = filePathInputRootDirectory; handler.Inputs = inputs; handler.SecureFiles = secureFiles; handler.TaskDirectory = taskDirectory; return(handler); }
private void ProcessPublishTestResultsCommand(IExecutionContext context, Dictionary <string, string> eventProperties, string data) { ArgUtil.NotNull(context, nameof(context)); _executionContext = context; LoadPublishTestResultsInputs(context, eventProperties, data); string teamProject = context.Variables.System_TeamProject; string owner = context.Variables.Build_RequestedFor; string buildUri = context.Variables.Build_BuildUri; int buildId = context.Variables.Build_BuildId ?? 0; string pullRequestTargetBranchName = context.Variables.System_PullRequest_TargetBranch; //Temporary fix to support publish in RM scenarios where there might not be a valid Build ID associated. //TODO: Make a cleaner fix after TCM User Story 401703 is completed. if (buildId == 0) { _platform = _configuration = null; } string releaseUri = null; string releaseEnvironmentUri = null; // Check to identify if we are in the Release management flow; if not, then release fields will be kept null while publishing to TCM if (!string.IsNullOrWhiteSpace(context.Variables.Release_ReleaseUri)) { releaseUri = context.Variables.Release_ReleaseUri; releaseEnvironmentUri = context.Variables.Release_ReleaseEnvironmentUri; } IResultReader resultReader = GetTestResultReader(_testRunner); TestRunContext runContext = new TestRunContext(owner, _platform, _configuration, buildId, buildUri, releaseUri, releaseEnvironmentUri); runContext.PullRequestTargetBranchName = pullRequestTargetBranchName; VssConnection connection = WorkerUtilities.GetVssConnection(_executionContext); var publisher = HostContext.GetService <ITestRunPublisher>(); publisher.InitializePublisher(context, connection, teamProject, resultReader); var commandContext = HostContext.CreateService <IAsyncCommandContext>(); commandContext.InitializeCommandContext(context, StringUtil.Loc("PublishTestResults")); if (_mergeResults) { commandContext.Task = PublishAllTestResultsToSingleTestRunAsync(_testResultFiles, publisher, buildId, runContext, resultReader.Name, context.CancellationToken); } else { commandContext.Task = PublishToNewTestRunPerTestResultFileAsync(_testResultFiles, publisher, runContext, resultReader.Name, PublishBatchSize, context.CancellationToken); } _executionContext.AsyncCommands.Add(commandContext); if (_isTestRunOutcomeFailed) { _executionContext.Result = TaskResult.Failed; _executionContext.Error(StringUtil.Loc("FailedTestsInResults")); } }
public async Task RunPluginTaskAsync(IExecutionContext context, string plugin, Dictionary <string, string> inputs, Dictionary <string, string> environment, Variables runtimeVariables, EventHandler <ProcessDataReceivedEventArgs> outputHandler) { ArgUtil.NotNullOrEmpty(plugin, nameof(plugin)); // Only allow plugins we defined if (!_taskPlugins.Contains(plugin)) { throw new NotSupportedException(plugin); } // Resolve the working directory. string workingDirectory = HostContext.GetDirectory(WellKnownDirectory.Work); ArgUtil.Directory(workingDirectory, nameof(workingDirectory)); // Agent.PluginHost string file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), $"Agent.PluginHost{Util.IOUtil.ExeExtension}"); ArgUtil.File(file, $"Agent.PluginHost{Util.IOUtil.ExeExtension}"); // Agent.PluginHost's arguments string arguments = $"task \"{plugin}\""; // construct plugin context AgentTaskPluginExecutionContext pluginContext = new AgentTaskPluginExecutionContext { Inputs = inputs, Repositories = context.Repositories, Endpoints = context.Endpoints }; // variables foreach (var publicVar in runtimeVariables.Public) { pluginContext.Variables[publicVar.Key] = publicVar.Value; } foreach (var publicVar in runtimeVariables.Private) { pluginContext.Variables[publicVar.Key] = new VariableValue(publicVar.Value, true); } // task variables (used by wrapper task) foreach (var publicVar in context.TaskVariables.Public) { pluginContext.TaskVariables[publicVar.Key] = publicVar.Value; } foreach (var publicVar in context.TaskVariables.Private) { pluginContext.TaskVariables[publicVar.Key] = new VariableValue(publicVar.Value, true); } using (var processInvoker = HostContext.CreateService <IProcessInvoker>()) { processInvoker.OutputDataReceived += outputHandler; processInvoker.ErrorDataReceived += outputHandler; // Execute the process. Exit code 0 should always be returned. // A non-zero exit code indicates infrastructural failure. // Task failure should be communicated over STDOUT using ## commands. await processInvoker.ExecuteAsync(workingDirectory : workingDirectory, fileName : file, arguments : arguments, environment : environment, requireExitCodeZero : true, outputEncoding : null, killProcessOnCancel : false, contentsToStandardIn : new List <string>() { JsonUtility.ToString(pluginContext) }, cancellationToken : context.CancellationToken); } }
private async Task ProcessPluginCommandAsync(IAsyncCommandContext context, AgentCommandPluginExecutionContext pluginContext, string plugin, Command command, CancellationToken token) { // Resolve the working directory. string workingDirectory = HostContext.GetDirectory(WellKnownDirectory.Work); ArgUtil.Directory(workingDirectory, nameof(workingDirectory)); // Agent.PluginHost string file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), $"Agent.PluginHost{Util.IOUtil.ExeExtension}"); // Agent.PluginHost's arguments string arguments = $"command \"{plugin}\""; // Execute the process. Exit code 0 should always be returned. // A non-zero exit code indicates infrastructural failure. // Any content coming from STDERR will indicate the command process failed. // We can't use ## command for plugin to communicate, since we are already processing ## command using (var processInvoker = HostContext.CreateService <IProcessInvoker>()) { object stderrLock = new object(); List <string> stderr = new List <string>(); processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) => { context.Output(e.Data); }; processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) => { lock (stderrLock) { stderr.Add(e.Data); }; };; int returnCode = await processInvoker.ExecuteAsync(workingDirectory : workingDirectory, fileName : file, arguments : arguments, environment : null, requireExitCodeZero : false, outputEncoding : null, killProcessOnCancel : false, contentsToStandardIn : new List <string>() { JsonUtility.ToString(pluginContext) }, cancellationToken : token); if (returnCode != 0) { context.Output(string.Join(Environment.NewLine, stderr)); throw new ProcessExitCodeException(returnCode, file, arguments); } else if (stderr.Count > 0) { throw new InvalidOperationException(string.Join(Environment.NewLine, stderr)); } else { // Everything works fine. // Return code is 0. // No STDERR comes out. } } }
public async Task RunAsync() { // Validate args. Trace.Entering(); ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext)); ArgUtil.NotNull(Inputs, nameof(Inputs)); ArgUtil.Directory(TaskDirectory, nameof(TaskDirectory)); // Resolve the target script. string target = GetTarget(); ArgUtil.NotNullOrEmpty(target, nameof(target)); string scriptFile = Path.Combine(TaskDirectory, target); ArgUtil.File(scriptFile, nameof(scriptFile)); // Determine the working directory. string workingDirectory = GetWorkingDirectory(); if (String.IsNullOrEmpty(workingDirectory)) { workingDirectory = Path.GetDirectoryName(scriptFile); } else { if (!Directory.Exists(workingDirectory)) { Directory.CreateDirectory(workingDirectory); } } // Copy the OM binaries into the legacy host folder. ExecutionContext.Output(StringUtil.Loc("PrepareTaskExecutionHandler")); IOUtil.CopyDirectory( source: HostContext.GetDirectory(WellKnownDirectory.ServerOM), target: HostContext.GetDirectory(WellKnownDirectory.LegacyPSHost), cancellationToken: ExecutionContext.CancellationToken); Trace.Info("Finished copying files."); // Add the legacy ps host environment variables. AddLegacyHostEnvironmentVariables(scriptFile: scriptFile, workingDirectory: workingDirectory); AddPrependPathToEnvironment(); // Invoke the process. using (var processInvoker = HostContext.CreateService <IProcessInvoker>()) { processInvoker.OutputDataReceived += OnDataReceived; processInvoker.ErrorDataReceived += OnDataReceived; try { String vstsPSHostExe = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.LegacyPSHost), "LegacyVSTSPowerShellHost.exe"); Int32 exitCode = await processInvoker.ExecuteAsync(workingDirectory : workingDirectory, fileName : vstsPSHostExe, arguments : "", environment : Environment, cancellationToken : ExecutionContext.CancellationToken); // the exit code from vstsPSHost.exe indicate how many error record we get during execution // -1 exit code means infrastructure failure of Host itself. // this is to match current handler's logic. if (exitCode > 0) { if (ExecutionContext.Result != null) { ExecutionContext.Debug($"Task result already set. Not failing due to error count ({exitCode})."); } else { // We fail task and add issue. ExecutionContext.Result = TaskResult.Failed; ExecutionContext.Error(StringUtil.Loc("PSScriptError", exitCode)); } } else if (exitCode < 0) { // We fail task and add issue. ExecutionContext.Result = TaskResult.Failed; ExecutionContext.Error(StringUtil.Loc("VSTSHostNonZeroReturn", exitCode)); } } finally { processInvoker.OutputDataReceived -= OnDataReceived; processInvoker.ErrorDataReceived -= OnDataReceived; } } }
//create worker manager, create message listener and start listening to the queue private async Task <int> RunAsync(AgentSettings settings, bool runOnce = false) { try { Trace.Info(nameof(RunAsync)); _listener = HostContext.GetService <IMessageListener>(); if (!await _listener.CreateSessionAsync(HostContext.AgentShutdownToken)) { return(Constants.Agent.ReturnCode.TerminatedError); } HostContext.WritePerfCounter("SessionCreated"); _term.WriteLine(StringUtil.Loc("ListenForJobs", DateTime.UtcNow)); IJobDispatcher jobDispatcher = null; CancellationTokenSource messageQueueLoopTokenSource = CancellationTokenSource.CreateLinkedTokenSource(HostContext.AgentShutdownToken); try { var notification = HostContext.GetService <IJobNotification>(); if (!String.IsNullOrEmpty(settings.NotificationSocketAddress)) { notification.StartClient(settings.NotificationSocketAddress, settings.MonitorSocketAddress); } else { notification.StartClient(settings.NotificationPipeName, settings.MonitorSocketAddress, HostContext.AgentShutdownToken); } // this is not a reliable way to disable auto update. // we need server side work to really enable the feature // https://github.com/Microsoft/vsts-agent/issues/446 (Feature: Allow agent / pool to opt out of automatic updates) bool disableAutoUpdate = !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("agent.disableupdate")); bool autoUpdateInProgress = false; Task <bool> selfUpdateTask = null; bool runOnceJobReceived = false; jobDispatcher = HostContext.CreateService <IJobDispatcher>(); while (!HostContext.AgentShutdownToken.IsCancellationRequested) { TaskAgentMessage message = null; bool skipMessageDeletion = false; try { Task <TaskAgentMessage> getNextMessage = _listener.GetNextMessageAsync(messageQueueLoopTokenSource.Token); if (autoUpdateInProgress) { Trace.Verbose("Auto update task running at backend, waiting for getNextMessage or selfUpdateTask to finish."); Task completeTask = await Task.WhenAny(getNextMessage, selfUpdateTask); if (completeTask == selfUpdateTask) { autoUpdateInProgress = false; if (await selfUpdateTask) { Trace.Info("Auto update task finished at backend, an agent update is ready to apply exit the current agent instance."); Trace.Info("Stop message queue looping."); messageQueueLoopTokenSource.Cancel(); try { await getNextMessage; } catch (Exception ex) { Trace.Info($"Ignore any exception after cancel message loop. {ex}"); } if (runOnce) { return(Constants.Agent.ReturnCode.RunOnceAgentUpdating); } else { return(Constants.Agent.ReturnCode.AgentUpdating); } } else { Trace.Info("Auto update task finished at backend, there is no available agent update needs to apply, continue message queue looping."); } } } if (runOnceJobReceived) { Trace.Verbose("One time used agent has start running its job, waiting for getNextMessage or the job to finish."); Task completeTask = await Task.WhenAny(getNextMessage, jobDispatcher.RunOnceJobCompleted.Task); if (completeTask == jobDispatcher.RunOnceJobCompleted.Task) { Trace.Info("Job has finished at backend, the agent will exit since it is running under onetime use mode."); Trace.Info("Stop message queue looping."); messageQueueLoopTokenSource.Cancel(); try { await getNextMessage; } catch (Exception ex) { Trace.Info($"Ignore any exception after cancel message loop. {ex}"); } return(Constants.Agent.ReturnCode.Success); } } message = await getNextMessage; //get next message HostContext.WritePerfCounter($"MessageReceived_{message.MessageType}"); if (string.Equals(message.MessageType, AgentRefreshMessage.MessageType, StringComparison.OrdinalIgnoreCase)) { if (disableAutoUpdate) { Trace.Info("Refresh message received, skip autoupdate since environment variable agent.disableupdate is set."); } else { if (autoUpdateInProgress == false) { autoUpdateInProgress = true; var agentUpdateMessage = JsonUtility.FromString <AgentRefreshMessage>(message.Body); var selfUpdater = HostContext.GetService <ISelfUpdater>(); selfUpdateTask = selfUpdater.SelfUpdate(agentUpdateMessage, jobDispatcher, !runOnce && HostContext.StartupType != StartupType.Service, HostContext.AgentShutdownToken); Trace.Info("Refresh message received, kick-off selfupdate background process."); } else { Trace.Info("Refresh message received, skip autoupdate since a previous autoupdate is already running."); } } } else if (string.Equals(message.MessageType, JobRequestMessageTypes.AgentJobRequest, StringComparison.OrdinalIgnoreCase) || string.Equals(message.MessageType, JobRequestMessageTypes.PipelineAgentJobRequest, StringComparison.OrdinalIgnoreCase)) { if (autoUpdateInProgress || runOnceJobReceived) { skipMessageDeletion = true; Trace.Info($"Skip message deletion for job request message '{message.MessageId}'."); } else { Pipelines.AgentJobRequestMessage pipelineJobMessage = null; switch (message.MessageType) { case JobRequestMessageTypes.AgentJobRequest: var legacyJobMessage = JsonUtility.FromString <AgentJobRequestMessage>(message.Body); pipelineJobMessage = Pipelines.AgentJobRequestMessageUtil.Convert(legacyJobMessage); break; case JobRequestMessageTypes.PipelineAgentJobRequest: pipelineJobMessage = JsonUtility.FromString <Pipelines.AgentJobRequestMessage>(message.Body); break; } jobDispatcher.Run(pipelineJobMessage, runOnce); if (runOnce) { Trace.Info("One time used agent received job message."); runOnceJobReceived = true; } } } else if (string.Equals(message.MessageType, JobCancelMessage.MessageType, StringComparison.OrdinalIgnoreCase)) { var cancelJobMessage = JsonUtility.FromString <JobCancelMessage>(message.Body); bool jobCancelled = jobDispatcher.Cancel(cancelJobMessage); skipMessageDeletion = (autoUpdateInProgress || runOnceJobReceived) && !jobCancelled; if (skipMessageDeletion) { Trace.Info($"Skip message deletion for cancellation message '{message.MessageId}'."); } } else { Trace.Error($"Received message {message.MessageId} with unsupported message type {message.MessageType}."); } } finally { if (!skipMessageDeletion && message != null) { try { await _listener.DeleteMessageAsync(message); } catch (Exception ex) { Trace.Error($"Catch exception during delete message from message queue. message id: {message.MessageId}"); Trace.Error(ex); } finally { message = null; } } } } } finally { if (jobDispatcher != null) { await jobDispatcher.ShutdownAsync(); } //TODO: make sure we don't mask more important exception await _listener.DeleteSessionAsync(); messageQueueLoopTokenSource.Dispose(); } } catch (TaskAgentAccessTokenExpiredException) { Trace.Info("Agent OAuth token has been revoked. Shutting down."); } return(Constants.Agent.ReturnCode.Success); }
public async Task RunAsync() { // Validate args. Trace.Entering(); ArgUtil.NotNull(Data, nameof(Data)); ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext)); ArgUtil.NotNull(Inputs, nameof(Inputs)); ArgUtil.NotNull(TaskDirectory, nameof(TaskDirectory)); // Update the env dictionary. AddVariablesToEnvironment(excludeNames: true, excludeSecrets: true); AddPrependPathToEnvironment(); // Get the command. ArgUtil.NotNullOrEmpty(Data.Target, nameof(Data.Target)); // TODO: WHICH the command? string command = Data.Target; // Determine whether the command is rooted. // TODO: If command begins and ends with a double-quote, trim quotes before making determination. Likewise when determining whether the file exists. bool isCommandRooted = false; try { // Path.IsPathRooted throws if illegal characters are in the path. isCommandRooted = Path.IsPathRooted(command); } catch (Exception ex) { Trace.Info($"Unable to determine whether the command is rooted: {ex.Message}"); } Trace.Info($"Command is rooted: {isCommandRooted}"); // Determine the working directory. string workingDirectory; if (!string.IsNullOrEmpty(Data.WorkingDirectory)) { workingDirectory = Data.WorkingDirectory; } else { if (isCommandRooted && File.Exists(command)) { workingDirectory = Path.GetDirectoryName(command); } else { workingDirectory = Path.Combine(TaskDirectory, "DefaultTaskWorkingDirectory"); } } ExecutionContext.Debug($"Working directory: '{workingDirectory}'"); Directory.CreateDirectory(workingDirectory); // Wrap the command in quotes if required. // // This is guess-work but is probably mostly accurate. The problem is that the command text // box is separate from the args text box. This implies to the user that we take care of quoting // for the command. // // The approach taken here is only to quote if it needs quotes. We should stay out of the way // as much as possible. Built-in shell commands will not work if they are quoted, e.g. RMDIR. if (command.Contains(" ") || command.Contains("%")) { if (!command.Contains("\"")) { command = StringUtil.Format("\"{0}\"", command); } } // Get the arguments. string arguments = Data.ArgumentFormat ?? string.Empty; // Get the fail on standard error flag. bool failOnStandardError = true; string failOnStandardErrorString; if (Inputs.TryGetValue("failOnStandardError", out failOnStandardErrorString)) { failOnStandardError = StringUtil.ConvertToBoolean(failOnStandardErrorString); } ExecutionContext.Debug($"Fail on standard error: '{failOnStandardError}'"); // Get the modify environment flag. _modifyEnvironment = StringUtil.ConvertToBoolean(Data.ModifyEnvironment); ExecutionContext.Debug($"Modify environment: '{_modifyEnvironment}'"); // Resolve cmd.exe. string cmdExe = System.Environment.GetEnvironmentVariable("ComSpec"); if (string.IsNullOrEmpty(cmdExe)) { cmdExe = "cmd.exe"; } // Format the input to be invoked from cmd.exe to enable built-in shell commands. For example, RMDIR. string cmdExeArgs; if (_modifyEnvironment) { // Format the command so the environment variables can be captured. cmdExeArgs = $"/c \"{command} {arguments} && echo {OutputDelimiter} && set \""; } else { cmdExeArgs = $"/c \"{command} {arguments}\""; } // Invoke the process. ExecutionContext.Debug($"{cmdExe} {cmdExeArgs}"); ExecutionContext.Command($"{command} {arguments}"); using (var processInvoker = HostContext.CreateService <IProcessInvoker>()) { processInvoker.OutputDataReceived += OnOutputDataReceived; if (failOnStandardError) { processInvoker.ErrorDataReceived += OnErrorDataReceived; } else { processInvoker.ErrorDataReceived += OnOutputDataReceived; } int exitCode = await processInvoker.ExecuteAsync( workingDirectory : workingDirectory, fileName : cmdExe, arguments : cmdExeArgs, environment : Environment, cancellationToken : ExecutionContext.CancellationToken); FlushErrorData(); // Fail on error count. if (_errorCount > 0) { if (ExecutionContext.Result != null) { Trace.Info($"Task result already set. Not failing due to error count ({_errorCount})."); } else { throw new Exception(StringUtil.Loc("ProcessCompletedWithCode0Errors1", exitCode, _errorCount)); } } // Fail on non-zero exit code. if (exitCode != 0) { throw new Exception(StringUtil.Loc("ProcessCompletedWithExitCode0", exitCode)); } } }
private async Task <int> DownloadStreams(IExecutionContext executionContext, Stream zipStream, string localFolderPath, string folderWithinStream, string relativePathWithinStream) { Trace.Entering(); int streamsDownloaded = 0; var fileSystemManager = HostContext.CreateService <IReleaseFileSystemManager>(); foreach (ZipEntryStream stream in GetZipEntryStreams(zipStream)) { try { // Remove leading '/'s if any var path = stream.FullName.TrimStart(ForwardSlash); Trace.Verbose($"Downloading {path}"); if (!string.IsNullOrWhiteSpace(folderWithinStream)) { var normalizedFolderWithInStream = folderWithinStream.TrimStart(ForwardSlash).TrimEnd(ForwardSlash) + ForwardSlash; // If this zip entry does not start with the expected folderName, skip it. if (!path.StartsWith(normalizedFolderWithInStream, StringComparison.OrdinalIgnoreCase)) { continue; } path = path.Substring(normalizedFolderWithInStream.Length); } if (!string.IsNullOrWhiteSpace(relativePathWithinStream) && !relativePathWithinStream.Equals(ForwardSlash.ToString()) && !relativePathWithinStream.Equals(Backslash.ToString())) { var normalizedRelativePath = relativePathWithinStream.Replace(Backslash, ForwardSlash).TrimStart(ForwardSlash).TrimEnd(ForwardSlash) + ForwardSlash; // Remove Blob Prefix path like "FabrikamFiber.DAL/bin/debug/" from the beginning of artifact full path if (!path.StartsWith(normalizedRelativePath, StringComparison.OrdinalIgnoreCase)) { continue; } path = path.Substring(normalizedRelativePath.Length); } await fileSystemManager.WriteStreamToFile(stream.ZipStream, Path.Combine(localFolderPath, path)); streamsDownloaded++; } finally { stream.ZipStream.Dispose(); } } if (streamsDownloaded == 0) { executionContext.Warning(StringUtil.Loc("RMArtifactEmpty")); } return(streamsDownloaded); }
public void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token) { // Validation Trace.Entering(); ArgUtil.NotNull(message, nameof(message)); ArgUtil.NotNull(message.Resources, nameof(message.Resources)); ArgUtil.NotNull(message.Variables, nameof(message.Variables)); ArgUtil.NotNull(message.Plan, nameof(message.Plan)); _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token); // Features Features = PlanUtil.GetFeatures(message.Plan); // Endpoints Endpoints = message.Resources.Endpoints; // SecureFiles SecureFiles = message.Resources.SecureFiles; // Repositories Repositories = message.Resources.Repositories; // JobSettings var checkouts = message.Steps?.Where(x => Pipelines.PipelineConstants.IsCheckoutTask(x)).ToList(); JobSettings = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase); JobSettings[WellKnownJobSettings.HasMultipleCheckouts] = Boolean.FalseString; if (checkouts != null && checkouts.Count > 0) { JobSettings[WellKnownJobSettings.HasMultipleCheckouts] = checkouts.Count > 1 ? Boolean.TrueString : Boolean.FalseString; var firstCheckout = checkouts.First() as Pipelines.TaskStep; if (firstCheckout != null && Repositories != null && firstCheckout.Inputs.TryGetValue(Pipelines.PipelineConstants.CheckoutTaskInputs.Repository, out string repoAlias)) { JobSettings[WellKnownJobSettings.FirstRepositoryCheckedOut] = repoAlias; var repo = Repositories.Find(r => String.Equals(r.Alias, repoAlias, StringComparison.OrdinalIgnoreCase)); if (repo != null) { repo.Properties.Set <bool>(RepositoryUtil.IsPrimaryRepository, true); } } } // Variables (constructor performs initial recursive expansion) List <string> warnings; Variables = new Variables(HostContext, message.Variables, out warnings); Variables.StringTranslator = TranslatePathForStepTarget; if (Variables.GetBoolean("agent.useWorkspaceId") == true) { try { // We need an identifier that represents which repos make up the workspace. // This allows similar jobs in the same definition to reuse that workspace and other jobs to have their own. JobSettings[WellKnownJobSettings.WorkspaceIdentifier] = GetWorkspaceIdentifier(message); } catch (Exception ex) { Trace.Warning($"Unable to generate workspace ID: {ex.Message}"); } } // Prepend Path PrependPath = new List <string>(); // Docker (JobContainer) string imageName = Variables.Get("_PREVIEW_VSTS_DOCKER_IMAGE"); if (string.IsNullOrEmpty(imageName)) { imageName = Environment.GetEnvironmentVariable("_PREVIEW_VSTS_DOCKER_IMAGE"); } Containers = new List <ContainerInfo>(); _defaultStepTarget = null; _currentStepTarget = null; if (!string.IsNullOrEmpty(imageName) && string.IsNullOrEmpty(message.JobContainer)) { var dockerContainer = new Pipelines.ContainerResource() { Alias = "vsts_container_preview" }; dockerContainer.Properties.Set("image", imageName); var defaultJobContainer = HostContext.CreateContainerInfo(dockerContainer); _defaultStepTarget = defaultJobContainer; Containers.Add(defaultJobContainer); } else if (!string.IsNullOrEmpty(message.JobContainer)) { var defaultJobContainer = HostContext.CreateContainerInfo(message.Resources.Containers.Single(x => string.Equals(x.Alias, message.JobContainer, StringComparison.OrdinalIgnoreCase))); _defaultStepTarget = defaultJobContainer; Containers.Add(defaultJobContainer); } else { _defaultStepTarget = new HostInfo(); } // Include other step containers var sidecarContainers = new HashSet <string>(message.JobSidecarContainers.Values, StringComparer.OrdinalIgnoreCase); foreach (var container in message.Resources.Containers.Where(x => !string.Equals(x.Alias, message.JobContainer, StringComparison.OrdinalIgnoreCase) && !sidecarContainers.Contains(x.Alias))) { Containers.Add(HostContext.CreateContainerInfo(container)); } // Docker (Sidecar Containers) SidecarContainers = new List <ContainerInfo>(); foreach (var sidecar in message.JobSidecarContainers) { var networkAlias = sidecar.Key; var containerResourceAlias = sidecar.Value; var containerResource = message.Resources.Containers.Single(c => string.Equals(c.Alias, containerResourceAlias, StringComparison.OrdinalIgnoreCase)); ContainerInfo containerInfo = HostContext.CreateContainerInfo(containerResource, isJobContainer: false); containerInfo.ContainerNetworkAlias = networkAlias; SidecarContainers.Add(containerInfo); } // Proxy variables var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>(); if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress)) { Variables.Set(Constants.Variables.Agent.ProxyUrl, agentWebProxy.ProxyAddress); Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY", string.Empty); if (!string.IsNullOrEmpty(agentWebProxy.ProxyUsername)) { Variables.Set(Constants.Variables.Agent.ProxyUsername, agentWebProxy.ProxyUsername); Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_USERNAME", string.Empty); } if (!string.IsNullOrEmpty(agentWebProxy.ProxyPassword)) { Variables.Set(Constants.Variables.Agent.ProxyPassword, agentWebProxy.ProxyPassword, true); Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_PASSWORD", string.Empty); } if (agentWebProxy.ProxyBypassList.Count > 0) { Variables.Set(Constants.Variables.Agent.ProxyBypassList, JsonUtility.ToString(agentWebProxy.ProxyBypassList)); } } // Certificate variables var agentCert = HostContext.GetService <IAgentCertificateManager>(); if (agentCert.SkipServerCertificateValidation) { Variables.Set(Constants.Variables.Agent.SslSkipCertValidation, bool.TrueString); } if (!string.IsNullOrEmpty(agentCert.CACertificateFile)) { Variables.Set(Constants.Variables.Agent.SslCAInfo, agentCert.CACertificateFile); } if (!string.IsNullOrEmpty(agentCert.ClientCertificateFile) && !string.IsNullOrEmpty(agentCert.ClientCertificatePrivateKeyFile) && !string.IsNullOrEmpty(agentCert.ClientCertificateArchiveFile)) { Variables.Set(Constants.Variables.Agent.SslClientCert, agentCert.ClientCertificateFile); Variables.Set(Constants.Variables.Agent.SslClientCertKey, agentCert.ClientCertificatePrivateKeyFile); Variables.Set(Constants.Variables.Agent.SslClientCertArchive, agentCert.ClientCertificateArchiveFile); if (!string.IsNullOrEmpty(agentCert.ClientCertificatePassword)) { Variables.Set(Constants.Variables.Agent.SslClientCertPassword, agentCert.ClientCertificatePassword, true); } } // Runtime option variables var runtimeOptions = HostContext.GetService <IConfigurationStore>().GetAgentRuntimeOptions(); if (runtimeOptions != null) { if (PlatformUtil.RunningOnWindows && runtimeOptions.GitUseSecureChannel) { Variables.Set(Constants.Variables.Agent.GitUseSChannel, runtimeOptions.GitUseSecureChannel.ToString()); } } // Job timeline record. InitializeTimelineRecord( timelineId: message.Timeline.Id, timelineRecordId: message.JobId, parentTimelineRecordId: null, recordType: ExecutionContextType.Job, displayName: message.JobDisplayName, refName: message.JobName, order: null); // The job timeline record's order is set by server. // Logger (must be initialized before writing warnings). _logger = HostContext.CreateService <IPagingLogger>(); _logger.Setup(_mainTimelineId, _record.Id); // Log warnings from recursive variable expansion. warnings?.ForEach(x => this.Warning(x)); // Verbosity (from system.debug). WriteDebug = Variables.System_Debug ?? false; // Hook up JobServerQueueThrottling event, we will log warning on server tarpit. _jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived; }
public async Task GetSourceAsync( IExecutionContext executionContext, ServiceEndpoint endpoint, CancellationToken cancellationToken) { // Validate args. ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(endpoint, nameof(endpoint)); // Create the tf command manager. var tf = HostContext.CreateService <ITeeTFCommandManager>(); tf.CancellationToken = cancellationToken; tf.Endpoint = endpoint; tf.ExecutionContext = executionContext; // Check if the administrator accepted the license terms of the TEE EULA when // configuring the agent. AgentSettings settings = HostContext.GetService <IConfigurationStore>().GetSettings(); if (settings.AcceptTeeEula) { // Check if the "tf eula -accept" command needs to be run for the current security user. bool skipEula = false; try { skipEula = tf.TestEulaAccepted(); } catch (Exception ex) { Trace.Error("Unexpected exception while testing whether the TEE EULA has been accepted for the current security user."); Trace.Error(ex); } if (!skipEula) { // Run the command "tf eula -accept". try { await tf.EulaAsync(); } catch (Exception ex) { Trace.Error(ex); executionContext.Warning(ex.Message); } } } // Get the TEE workspaces. TeeWorkspace[] teeWorkspaces = await tf.WorkspacesAsync(); // Determine the workspace name. string buildDirectory = executionContext.Variables.Agent_BuildDirectory; ArgUtil.NotNullOrEmpty(buildDirectory, nameof(buildDirectory)); string workspaceName = $"ws_{Path.GetFileName(buildDirectory)}_{settings.AgentId}"; executionContext.Variables.Set(Constants.Variables.Build.RepoTfvcWorkspace, workspaceName); // Get the definition mappings. TfsVCTeeWorkspaceMapping[] definitionMappings = JsonConvert.DeserializeObject <TfsVCTeeWorkspaceMappings>(endpoint.Data["tfvcWorkspaceMapping"])?.Mappings; // Determine the sources directory. string sourcesDirectory = executionContext.Variables.Build_SourcesDirectory; ArgUtil.NotNullOrEmpty(sourcesDirectory, nameof(sourcesDirectory)); // Attempt to re-use an existing workspace if clean=false. TeeWorkspace existingTeeWorkspace = null; bool clean = endpoint.Data.ContainsKey(WellKnownEndpointData.Clean) && StringUtil.ConvertToBoolean(endpoint.Data[WellKnownEndpointData.Clean], defaultValue: false); if (!clean) { existingTeeWorkspace = MatchExactWorkspace( teeWorkspaces: teeWorkspaces, name: workspaceName, definitionMappings: definitionMappings, sourcesDirectory: sourcesDirectory); // Undo any pending changes. if (existingTeeWorkspace != null) { TeeStatus teeStatus = await tf.StatusAsync(workspaceName); if (teeStatus?.PendingChanges?.Any() ?? false) { await tf.UndoAsync(sourcesDirectory); // Cleanup remaining files/directories from pend adds. teeStatus.PendingChanges .Where(x => string.Equals(x.ChangeType, "add", StringComparison.OrdinalIgnoreCase)) .OrderByDescending(x => x.LocalItem) // Sort descending so nested items are deleted before their parent is deleted. .ToList() .ForEach(x => { executionContext.Output(StringUtil.Loc("Deleting", x.LocalItem)); IOUtil.Delete(x.LocalItem, cancellationToken); }); } } } // Create a new workspace. if (existingTeeWorkspace == null) { // Remove any conflicting TEE workspaces. await RemoveConflictingWorkspacesAsync( tf : tf, teeWorkspaces : teeWorkspaces, name : workspaceName, directory : sourcesDirectory); // Recreate the sources directory. executionContext.Debug($"Deleting: '{sourcesDirectory}'."); IOUtil.DeleteDirectory(sourcesDirectory, cancellationToken); Directory.CreateDirectory(sourcesDirectory); // Create the TEE workspace. await tf.WorkspaceNewAsync(workspaceName); // Sort the definition mappings. definitionMappings = (definitionMappings ?? new TfsVCTeeWorkspaceMapping[0]) .OrderBy(x => NormalizeServerPath(x.ServerPath)?.Length ?? 0) // By server path length. .ToArray() ?? new TfsVCTeeWorkspaceMapping[0]; // Add the definition mappings to the TEE workspace. foreach (TfsVCTeeWorkspaceMapping definitionMapping in definitionMappings) { switch (definitionMapping.MappingType) { case TfsVCTeeMappingType.Cloak: // Add the cloak. await tf.WorkfoldCloakAsync( workspace : workspaceName, serverPath : definitionMapping.ServerPath); break; case TfsVCTeeMappingType.Map: // Add the mapping. await tf.WorkfoldMapAsync( workspace : workspaceName, serverPath : definitionMapping.ServerPath, localPath : ResolveMappingLocalPath(definitionMapping, sourcesDirectory)); break; default: throw new NotSupportedException(); } } } // Get. await tf.GetAsync( version : executionContext.Variables.Build_SourceVersion, directory : sourcesDirectory); string shelvesetName = executionContext.Variables.Build_SourceTfvcShelveset; if (!string.IsNullOrEmpty(shelvesetName)) { // Get the shelveset details. TeeShelveset teeShelveset = null; string gatedShelvesetName = executionContext.Variables.Build_GatedShelvesetName; if (!string.IsNullOrEmpty(gatedShelvesetName)) { teeShelveset = await tf.ShelvesetsAsync(workspace : workspaceName, shelveset : shelvesetName); // The command throws if the shelveset is not found. // This assertion should never fail. ArgUtil.NotNull(teeShelveset, nameof(teeShelveset)); } // Unshelve. await tf.UnshelveAsync(workspace : workspaceName, shelveset : shelvesetName); if (!string.IsNullOrEmpty(gatedShelvesetName)) { // Create the comment file for reshelve. StringBuilder comment = new StringBuilder(teeShelveset.Comment ?? string.Empty); if (!(executionContext.Variables.Build_GatedRunCI ?? true)) { if (comment.Length > 0) { comment.AppendLine(); } comment.Append(Constants.Build.NoCICheckInComment); } string commentFile = null; try { commentFile = Path.GetTempFileName(); // TODO: FIGURE OUT WHAT ENCODING TF EXPECTS File.WriteAllText(path: commentFile, contents: comment.ToString()); // Reshelve. await tf.ShelveAsync( shelveset : gatedShelvesetName, directory : sourcesDirectory, commentFile : commentFile); } finally { // Cleanup the comment file. if (File.Exists(commentFile)) { File.Delete(commentFile); } } } } }
public async Task <int> LocalRunAsync(CommandSettings command, CancellationToken token) { Trace.Info(nameof(LocalRunAsync)); // Warn preview. _term.WriteLine("This command is currently in preview. The interface and behavior will change in a future version."); if (!command.Unattended) { _term.WriteLine("Press Enter to continue."); _term.ReadLine(); } HostContext.RunMode = RunMode.Local; // Resolve the YAML file path. string ymlFile = command.GetYml(); if (string.IsNullOrEmpty(ymlFile)) { string[] ymlFiles = Directory.GetFiles(Directory.GetCurrentDirectory()) .Where((string filePath) => { return(filePath.EndsWith(".yml", IOUtil.FilePathStringComparison)); }) .ToArray(); if (ymlFiles.Length > 1) { throw new Exception($"More than one .yml file exists in the current directory. Specify which file to use via the --'{Constants.Agent.CommandLine.Args.Yml}' command line argument."); } ymlFile = ymlFiles.FirstOrDefault(); } if (string.IsNullOrEmpty(ymlFile)) { throw new Exception($"Unable to find a .yml file in the current directory. Specify which file to use via the --'{Constants.Agent.CommandLine.Args.Yml}' command line argument."); } // Load the YAML file. var parseOptions = new ParseOptions { MaxFiles = 10, MustacheEvaluationMaxResultLength = 512 * 1024, // 512k string length MustacheEvaluationTimeout = TimeSpan.FromSeconds(10), MustacheMaxDepth = 5, }; var pipelineParser = new PipelineParser(new PipelineTraceWriter(), new PipelineFileProvider(), parseOptions); if (command.WhatIf) { pipelineParser.DeserializeAndSerialize( defaultRoot: Directory.GetCurrentDirectory(), path: ymlFile, mustacheContext: null, cancellationToken: HostContext.AgentShutdownToken); return(Constants.Agent.ReturnCode.Success); } YamlContracts.Process process = pipelineParser.LoadInternal( defaultRoot: Directory.GetCurrentDirectory(), path: ymlFile, mustacheContext: null, cancellationToken: HostContext.AgentShutdownToken); ArgUtil.NotNull(process, nameof(process)); // Verify the current directory is the root of a git repo. string repoDirectory = Directory.GetCurrentDirectory(); if (!Directory.Exists(Path.Combine(repoDirectory, ".git"))) { throw new Exception("Unable to run the build locally. The command must be executed from the root directory of a local git repository."); } // Verify at least one phase was found. if (process.Phases == null || process.Phases.Count == 0) { throw new Exception($"No phases or steps were discovered from the file: '{ymlFile}'"); } // Filter the phases. string phaseName = command.GetPhase(); if (!string.IsNullOrEmpty(phaseName)) { process.Phases = process.Phases .Cast <YamlContracts.Phase>() .Where(x => string.Equals(x.Name, phaseName, StringComparison.OrdinalIgnoreCase)) .Cast <YamlContracts.IPhase>() .ToList(); if (process.Phases.Count == 0) { throw new Exception($"Phase '{phaseName}' not found."); } } // Verify a phase was specified if more than one phase was found. if (process.Phases.Count > 1) { throw new Exception($"More than one phase was discovered. Use the --{Constants.Agent.CommandLine.Args.Phase} argument to specify a phase."); } // Get the matrix. var phase = process.Phases[0] as YamlContracts.Phase; var queueTarget = phase.Target as QueueTarget; // Filter to a specific matrix. string matrixName = command.GetMatrix(); if (!string.IsNullOrEmpty(matrixName)) { if (queueTarget?.Matrix != null) { queueTarget.Matrix = queueTarget.Matrix.Keys .Where(x => string.Equals(x, matrixName, StringComparison.OrdinalIgnoreCase)) .ToDictionary(keySelector: x => x, elementSelector: x => queueTarget.Matrix[x]); } if (queueTarget?.Matrix == null || queueTarget.Matrix.Count == 0) { throw new Exception($"Job configuration matrix '{matrixName}' not found."); } } // Verify a matrix was specified if more than one matrix was found. if (queueTarget?.Matrix != null && queueTarget.Matrix.Count > 1) { throw new Exception($"More than one job configuration matrix was discovered. Use the --{Constants.Agent.CommandLine.Args.Matrix} argument to specify a matrix."); } // Get the URL - required if missing tasks. string url = command.GetUrl(suppressPromptIfEmpty: true); if (string.IsNullOrEmpty(url)) { if (!TestAllTasksCached(process, token)) { url = command.GetUrl(suppressPromptIfEmpty: false); } } if (!string.IsNullOrEmpty(url)) { // Initialize and store the HTTP client. var credentialManager = HostContext.GetService <ICredentialManager>(); // Defaults to PAT authentication. string defaultAuthType = Constants.Configuration.PAT; string authType = command.GetAuth(defaultValue: defaultAuthType); ICredentialProvider provider = credentialManager.GetCredentialProvider(authType); provider.EnsureCredential(HostContext, command, url); _taskStore.HttpClient = new TaskAgentHttpClient(new Uri(url), provider.GetVssCredentials(HostContext)); } var configStore = HostContext.GetService <IConfigurationStore>(); AgentSettings settings = configStore.GetSettings(); // Create job message. JobInfo job = (await ConvertToJobMessagesAsync(process, repoDirectory, token)).Single(); IJobDispatcher jobDispatcher = null; try { jobDispatcher = HostContext.CreateService <IJobDispatcher>(); job.RequestMessage.Environment.Variables[Constants.Variables.Agent.RunMode] = RunMode.Local.ToString(); jobDispatcher.Run(Pipelines.AgentJobRequestMessageUtil.Convert(job.RequestMessage)); Task jobDispatch = jobDispatcher.WaitAsync(token); if (!Task.WaitAll(new[] { jobDispatch }, job.Timeout)) { jobDispatcher.Cancel(job.CancelMessage); // Finish waiting on the job dispatch task. The call to jobDispatcher.WaitAsync dequeues // the job dispatch task. In the cancel flow, we need to continue awaiting the task instance // (queue is now empty). await jobDispatch; } // Translate the job result to an agent return code. TaskResult jobResult = jobDispatcher.GetLocalRunJobResult(job.RequestMessage); switch (jobResult) { case TaskResult.Succeeded: case TaskResult.SucceededWithIssues: return(Constants.Agent.ReturnCode.Success); default: return(Constants.Agent.ReturnCode.TerminatedError); } } finally { if (jobDispatcher != null) { await jobDispatcher.ShutdownAsync(); } } }
private async Task RunAsync(Pipelines.AgentJobRequestMessage message, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken, CancellationToken workerCancelTimeoutKillToken) { if (previousJobDispatch != null) { Trace.Verbose($"Make sure the previous job request {previousJobDispatch.JobId} has successfully finished on worker."); await EnsureDispatchFinished(previousJobDispatch); } else { Trace.Verbose($"This is the first job request."); } var term = HostContext.GetService <ITerminal>(); term.WriteLine(StringUtil.Loc("RunningJob", DateTime.UtcNow, message.JobDisplayName)); // first job request renew succeed. TaskCompletionSource <int> firstJobRequestRenewed = new TaskCompletionSource <int>(); var notification = HostContext.GetService <IJobNotification>(); // lock renew cancellation token. using (var lockRenewalTokenSource = new CancellationTokenSource()) using (var workerProcessCancelTokenSource = new CancellationTokenSource()) { long requestId = message.RequestId; Guid lockToken = Guid.Empty; // lockToken has never been used, keep this here of compat // start renew job request Trace.Info($"Start renew job request {requestId} for job {message.JobId}."); Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, firstJobRequestRenewed, lockRenewalTokenSource.Token); // wait till first renew succeed or job request is canceled // not even start worker if the first renew fail await Task.WhenAny(firstJobRequestRenewed.Task, renewJobRequest, Task.Delay(-1, jobRequestCancellationToken)); if (renewJobRequest.IsCompleted) { // renew job request task complete means we run out of retry for the first job request renew. Trace.Info($"Unable to renew job request for job {message.JobId} for the first time, stop dispatching job to worker."); return; } if (jobRequestCancellationToken.IsCancellationRequested) { Trace.Info($"Stop renew job request for job {message.JobId}."); // stop renew lock lockRenewalTokenSource.Cancel(); // renew job request should never blows up. await renewJobRequest; // complete job request with result Cancelled await CompleteJobRequestAsync(_poolId, message, lockToken, TaskResult.Canceled); return; } HostContext.WritePerfCounter($"JobRequestRenewed_{requestId.ToString()}"); Task <int> workerProcessTask = null; object _outputLock = new object(); List <string> workerOutput = new List <string>(); using (var processChannel = HostContext.CreateService <IProcessChannel>()) using (var processInvoker = HostContext.CreateService <IProcessInvoker>()) { // Start the process channel. // It's OK if StartServer bubbles an execption after the worker process has already started. // The worker will shutdown after 30 seconds if it hasn't received the job message. processChannel.StartServer( // Delegate to start the child process. startProcess: (string pipeHandleOut, string pipeHandleIn) => { // Validate args. ArgUtil.NotNullOrEmpty(pipeHandleOut, nameof(pipeHandleOut)); ArgUtil.NotNullOrEmpty(pipeHandleIn, nameof(pipeHandleIn)); if (HostContext.RunMode == RunMode.Normal) { // Save STDOUT from worker, worker will use STDOUT report unhandle exception. processInvoker.OutputDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stdout) { if (!string.IsNullOrEmpty(stdout.Data)) { lock (_outputLock) { workerOutput.Add(stdout.Data); } } }; // Save STDERR from worker, worker will use STDERR on crash. processInvoker.ErrorDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stderr) { if (!string.IsNullOrEmpty(stderr.Data)) { lock (_outputLock) { workerOutput.Add(stderr.Data); } } }; } else if (HostContext.RunMode == RunMode.Local) { processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) => Console.WriteLine(e.Data); processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) => Console.WriteLine(e.Data); } // Start the child process. HostContext.WritePerfCounter("StartingWorkerProcess"); var assemblyDirectory = HostContext.GetDirectory(WellKnownDirectory.Bin); string workerFileName = Path.Combine(assemblyDirectory, _workerProcessName); workerProcessTask = processInvoker.ExecuteAsync( workingDirectory: assemblyDirectory, fileName: workerFileName, arguments: "spawnclient " + pipeHandleOut + " " + pipeHandleIn, environment: null, requireExitCodeZero: false, outputEncoding: null, killProcessOnCancel: true, cancellationToken: workerProcessCancelTokenSource.Token); }); // Send the job request message. // Kill the worker process if sending the job message times out. The worker // process may have successfully received the job message. try { Trace.Info($"Send job request message to worker for job {message.JobId}."); HostContext.WritePerfCounter($"AgentSendingJobToWorker_{message.JobId}"); using (var csSendJobRequest = new CancellationTokenSource(_channelTimeout)) { await processChannel.SendAsync( messageType : MessageType.NewJobRequest, body : JsonUtility.ToString(message), cancellationToken : csSendJobRequest.Token); } } catch (OperationCanceledException) { // message send been cancelled. // timeout 30 sec. kill worker. Trace.Info($"Job request message sending for job {message.JobId} been cancelled, kill running worker."); workerProcessCancelTokenSource.Cancel(); try { await workerProcessTask; } catch (OperationCanceledException) { Trace.Info("worker process has been killed."); } Trace.Info($"Stop renew job request for job {message.JobId}."); // stop renew lock lockRenewalTokenSource.Cancel(); // renew job request should never blows up. await renewJobRequest; // not finish the job request since the job haven't run on worker at all, we will not going to set a result to server. return; } // we get first jobrequest renew succeed and start the worker process with the job message. // send notification to machine provisioner. await notification.JobStarted(message.JobId); HostContext.WritePerfCounter($"SentJobToWorker_{requestId.ToString()}"); try { TaskResult resultOnAbandonOrCancel = TaskResult.Succeeded; // wait for renewlock, worker process or cancellation token been fired. var completedTask = await Task.WhenAny(renewJobRequest, workerProcessTask, Task.Delay(-1, jobRequestCancellationToken)); if (completedTask == workerProcessTask) { // worker finished successfully, complete job request with result, attach unhandled exception reported by worker, stop renew lock, job has finished. int returnCode = await workerProcessTask; Trace.Info($"Worker finished for job {message.JobId}. Code: " + returnCode); string detailInfo = null; if (!TaskResultUtil.IsValidReturnCode(returnCode)) { detailInfo = string.Join(Environment.NewLine, workerOutput); Trace.Info($"Return code {returnCode} indicate worker encounter an unhandled exception or app crash, attach worker stdout/stderr to JobRequest result."); await LogWorkerProcessUnhandledException(message, detailInfo); } TaskResult result = TaskResultUtil.TranslateFromReturnCode(returnCode); Trace.Info($"finish job request for job {message.JobId} with result: {result}"); term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobDisplayName, result)); Trace.Info($"Stop renew job request for job {message.JobId}."); // stop renew lock lockRenewalTokenSource.Cancel(); // renew job request should never blows up. await renewJobRequest; // complete job request await CompleteJobRequestAsync(_poolId, message, lockToken, result, detailInfo); // print out unhandled exception happened in worker after we complete job request. // when we run out of disk space, report back to server has higher priority. if (!string.IsNullOrEmpty(detailInfo)) { Trace.Error("Unhandled exception happened in worker:"); Trace.Error(detailInfo); } return; } else if (completedTask == renewJobRequest) { resultOnAbandonOrCancel = TaskResult.Abandoned; } else { resultOnAbandonOrCancel = TaskResult.Canceled; } // renew job request completed or job request cancellation token been fired for RunAsync(jobrequestmessage) // cancel worker gracefully first, then kill it after worker cancel timeout try { Trace.Info($"Send job cancellation message to worker for job {message.JobId}."); using (var csSendCancel = new CancellationTokenSource(_channelTimeout)) { var messageType = MessageType.CancelRequest; if (HostContext.AgentShutdownToken.IsCancellationRequested) { switch (HostContext.AgentShutdownReason) { case ShutdownReason.UserCancelled: messageType = MessageType.AgentShutdown; break; case ShutdownReason.OperatingSystemShutdown: messageType = MessageType.OperatingSystemShutdown; break; } } await processChannel.SendAsync( messageType : messageType, body : string.Empty, cancellationToken : csSendCancel.Token); } } catch (OperationCanceledException) { // message send been cancelled. Trace.Info($"Job cancel message sending for job {message.JobId} been cancelled, kill running worker."); workerProcessCancelTokenSource.Cancel(); try { await workerProcessTask; } catch (OperationCanceledException) { Trace.Info("worker process has been killed."); } } // wait worker to exit // if worker doesn't exit within timeout, then kill worker. completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, workerCancelTimeoutKillToken)); // worker haven't exit within cancellation timeout. if (completedTask != workerProcessTask) { Trace.Info($"worker process for job {message.JobId} haven't exit within cancellation timout, kill running worker."); workerProcessCancelTokenSource.Cancel(); try { await workerProcessTask; } catch (OperationCanceledException) { Trace.Info("worker process has been killed."); } } Trace.Info($"finish job request for job {message.JobId} with result: {resultOnAbandonOrCancel}"); term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobDisplayName, resultOnAbandonOrCancel)); // complete job request with cancel result, stop renew lock, job has finished. Trace.Info($"Stop renew job request for job {message.JobId}."); // stop renew lock lockRenewalTokenSource.Cancel(); // renew job request should never blows up. await renewJobRequest; // complete job request await CompleteJobRequestAsync(_poolId, message, lockToken, resultOnAbandonOrCancel); } finally { // This should be the last thing to run so we don't notify external parties until actually finished await notification.JobCompleted(message.JobId); } } } }
private async Task <string> GitAsync(string arguments, CancellationToken token) { // Resolve the location of git. if (_gitPath == null) { if (PlatformUtil.RunningOnWindows) { _gitPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "git", "cmd", $"git{IOUtil.ExeExtension}"); ArgUtil.File(_gitPath, nameof(_gitPath)); } else { _gitPath = WhichUtil.Which("git", require: true); } } // Prepare the environment variables to overlay. var overlayEnvironment = new Dictionary <string, string>(StringComparer.Ordinal); overlayEnvironment["GIT_TERMINAL_PROMPT"] = "0"; // Skip any GIT_TRACE variable since GIT_TRACE will affect ouput from every git command. // This will fail the parse logic for detect git version, remote url, etc. // Ex. // SET GIT_TRACE=true // git version // 11:39:58.295959 git.c:371 trace: built-in: git 'version' // git version 2.11.1.windows.1 IDictionary currentEnvironment = Environment.GetEnvironmentVariables(); foreach (DictionaryEntry entry in currentEnvironment) { string key = entry.Key as string ?? string.Empty; if (string.Equals(key, "GIT_TRACE", StringComparison.OrdinalIgnoreCase) || key.StartsWith("GIT_TRACE_", StringComparison.OrdinalIgnoreCase)) { overlayEnvironment[key] = string.Empty; } } // Run git and return the output from the streams. var output = new StringBuilder(); var processInvoker = HostContext.CreateService <IProcessInvoker>(); Console.WriteLine(); Console.WriteLine($"git {arguments}"); processInvoker.OutputDataReceived += delegate(object sender, ProcessDataReceivedEventArgs message) { output.AppendLine(message.Data); Console.WriteLine(message.Data); }; processInvoker.ErrorDataReceived += delegate(object sender, ProcessDataReceivedEventArgs message) { output.AppendLine(message.Data); Console.WriteLine(message.Data); }; Encoding encoding = null; if (PlatformUtil.RunningOnWindows) { encoding = Encoding.UTF8; } await processInvoker.ExecuteAsync( workingDirectory : Directory.GetCurrentDirectory(), fileName : _gitPath, arguments : arguments, environment : overlayEnvironment, requireExitCodeZero : true, outputEncoding : encoding, cancellationToken : token); string result = output.ToString().Trim(); ArgUtil.NotNullOrEmpty(result, nameof(result)); return(result); }
//create worker manager, create message listener and start listening to the queue private async Task <int> RunAsync(RunnerSettings settings, bool runOnce = false) { try { Trace.Info(nameof(RunAsync)); _listener = HostContext.GetService <IMessageListener>(); if (!await _listener.CreateSessionAsync(HostContext.RunnerShutdownToken)) { return(Constants.Runner.ReturnCode.TerminatedError); } HostContext.WritePerfCounter("SessionCreated"); _term.WriteLine($"{DateTime.UtcNow:u}: Listening for Jobs"); IJobDispatcher jobDispatcher = null; CancellationTokenSource messageQueueLoopTokenSource = CancellationTokenSource.CreateLinkedTokenSource(HostContext.RunnerShutdownToken); try { var notification = HostContext.GetService <IJobNotification>(); notification.StartClient(settings.MonitorSocketAddress); bool autoUpdateInProgress = false; Task <bool> selfUpdateTask = null; bool runOnceJobReceived = false; jobDispatcher = HostContext.CreateService <IJobDispatcher>(); while (!HostContext.RunnerShutdownToken.IsCancellationRequested) { TaskAgentMessage message = null; bool skipMessageDeletion = false; try { Task <TaskAgentMessage> getNextMessage = _listener.GetNextMessageAsync(messageQueueLoopTokenSource.Token); if (autoUpdateInProgress) { Trace.Verbose("Auto update task running at backend, waiting for getNextMessage or selfUpdateTask to finish."); Task completeTask = await Task.WhenAny(getNextMessage, selfUpdateTask); if (completeTask == selfUpdateTask) { autoUpdateInProgress = false; if (await selfUpdateTask) { Trace.Info("Auto update task finished at backend, an runner update is ready to apply exit the current runner instance."); Trace.Info("Stop message queue looping."); messageQueueLoopTokenSource.Cancel(); try { await getNextMessage; } catch (Exception ex) { Trace.Info($"Ignore any exception after cancel message loop. {ex}"); } if (runOnce) { return(Constants.Runner.ReturnCode.RunOnceRunnerUpdating); } else { return(Constants.Runner.ReturnCode.RunnerUpdating); } } else { Trace.Info("Auto update task finished at backend, there is no available runner update needs to apply, continue message queue looping."); } } } if (runOnceJobReceived) { Trace.Verbose("One time used runner has start running its job, waiting for getNextMessage or the job to finish."); Task completeTask = await Task.WhenAny(getNextMessage, jobDispatcher.RunOnceJobCompleted.Task); if (completeTask == jobDispatcher.RunOnceJobCompleted.Task) { Trace.Info("Job has finished at backend, the runner will exit since it is running under onetime use mode."); Trace.Info("Stop message queue looping."); messageQueueLoopTokenSource.Cancel(); try { await getNextMessage; } catch (Exception ex) { Trace.Info($"Ignore any exception after cancel message loop. {ex}"); } return(Constants.Runner.ReturnCode.Success); } } message = await getNextMessage; //get next message HostContext.WritePerfCounter($"MessageReceived_{message.MessageType}"); if (string.Equals(message.MessageType, AgentRefreshMessage.MessageType, StringComparison.OrdinalIgnoreCase)) { if (autoUpdateInProgress == false) { autoUpdateInProgress = true; var runnerUpdateMessage = JsonUtility.FromString <AgentRefreshMessage>(message.Body); var selfUpdater = HostContext.GetService <ISelfUpdater>(); selfUpdateTask = selfUpdater.SelfUpdate(runnerUpdateMessage, jobDispatcher, !runOnce && HostContext.StartupType != StartupType.Service, HostContext.RunnerShutdownToken); Trace.Info("Refresh message received, kick-off selfupdate background process."); } else { Trace.Info("Refresh message received, skip autoupdate since a previous autoupdate is already running."); } } else if (string.Equals(message.MessageType, JobRequestMessageTypes.PipelineAgentJobRequest, StringComparison.OrdinalIgnoreCase)) { if (autoUpdateInProgress || runOnceJobReceived) { skipMessageDeletion = true; Trace.Info($"Skip message deletion for job request message '{message.MessageId}'."); } else { var jobMessage = StringUtil.ConvertFromJson <Pipelines.AgentJobRequestMessage>(message.Body); jobDispatcher.Run(jobMessage, runOnce); if (runOnce) { Trace.Info("One time used runner received job message."); runOnceJobReceived = true; } } } else if (string.Equals(message.MessageType, JobCancelMessage.MessageType, StringComparison.OrdinalIgnoreCase)) { var cancelJobMessage = JsonUtility.FromString <JobCancelMessage>(message.Body); bool jobCancelled = jobDispatcher.Cancel(cancelJobMessage); skipMessageDeletion = (autoUpdateInProgress || runOnceJobReceived) && !jobCancelled; if (skipMessageDeletion) { Trace.Info($"Skip message deletion for cancellation message '{message.MessageId}'."); } } else { Trace.Error($"Received message {message.MessageId} with unsupported message type {message.MessageType}."); } } finally { if (!skipMessageDeletion && message != null) { try { await _listener.DeleteMessageAsync(message); } catch (Exception ex) { Trace.Error($"Catch exception during delete message from message queue. message id: {message.MessageId}"); Trace.Error(ex); } finally { message = null; } } } } } finally { if (jobDispatcher != null) { await jobDispatcher.ShutdownAsync(); } //TODO: make sure we don't mask more important exception await _listener.DeleteSessionAsync(); messageQueueLoopTokenSource.Dispose(); } } catch (TaskAgentAccessTokenExpiredException) { Trace.Info("Runner OAuth token has been revoked. Shutting down."); } return(Constants.Runner.ReturnCode.Success); }
public async Task RunAsync() { // Validate args. Trace.Entering(); ArgUtil.NotNull(Data, nameof(Data)); ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext)); ArgUtil.NotNull(Inputs, nameof(Inputs)); ArgUtil.Directory(TaskDirectory, nameof(TaskDirectory)); // Update the env dictionary. AddVariablesToEnvironment(excludeNames: true, excludeSecrets: true); AddPrependPathToEnvironment(); // Add the access token to the environment variables, if the access token is set. if (!string.IsNullOrEmpty(AccessToken)) { string formattedKey = Constants.Variables.System.AccessToken.Replace('.', '_').Replace(' ', '_').ToUpperInvariant(); AddEnvironmentVariable(formattedKey, AccessToken); } // Determine whether to fail on STDERR. _failOnStandardError = StringUtil.ConvertToBoolean(Data.FailOnStandardError, true); // Default to true. // Get the script file. string scriptFile = null; try { if (string.Equals(Data.ScriptType, InlineScriptType, StringComparison.OrdinalIgnoreCase)) { // TODO: Write this file under the _work folder and clean it up at the beginning of the next build? // Write the inline script to a temp file. string tempDirectory = Path.GetTempPath(); ArgUtil.Directory(tempDirectory, nameof(tempDirectory)); scriptFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.ps1"); Trace.Info("Writing inline script to temp file: '{0}'", scriptFile); File.WriteAllText(scriptFile, Data.InlineScript ?? string.Empty, Encoding.UTF8); } else { // TODO: If not rooted, WHICH the file if it doesn't contain any slashes. // Assert the target file. ArgUtil.NotNullOrEmpty(Data.Target, nameof(Data.Target)); scriptFile = Data.Target; } // Define the nested expression to invoke the user-specified script file and arguments. // Use the dot operator (".") to run the script in the same scope. string nestedExpression = StringUtil.Format( ". '{0}' {1}", scriptFile.Trim('"').Replace("'", "''"), Data.ArgumentFormat); // Craft the args to pass to PowerShell.exe. The user-defined expression is jammed in // as an encrypted base 64 string to a wrapper command. This solves a couple problems: // 1) Avoids quoting issues by jamming all of the user input into a base-64 encoded. // 2) Handles setting the exit code. // // The goal here is to jam everything into a base 64 encoded string so that quoting // issues can be avoided. The data needs to be encrypted because base 64 encoding the // data circumvents the logger's secret-masking behavior. string entropy; string powerShellExeArgs = StringUtil.Format( "-NoLogo -Sta -NoProfile -NonInteractive -ExecutionPolicy Unrestricted -Command \"try {{ $null = [System.Security.Cryptography.ProtectedData] }} catch {{ Write-Verbose 'Adding assemly: System.Security' ; Add-Type -AssemblyName 'System.Security' ; $null = [System.Security.Cryptography.ProtectedData] ; $Error.Clear() }} ; Invoke-Expression -Command ([System.Text.Encoding]::UTF8.GetString([System.Security.Cryptography.ProtectedData]::Unprotect([System.Convert]::FromBase64String('{0}'), [System.Convert]::FromBase64String('{1}'), [System.Security.Cryptography.DataProtectionScope]::CurrentUser))) ; if (!(Test-Path -LiteralPath variable:\\LastExitCode)) {{ Write-Verbose 'Last exit code is not set.' }} else {{ Write-Verbose ('$LastExitCode: {{0}}' -f $LastExitCode) ; exit $LastExitCode }}\"", Encrypt(nestedExpression, out entropy), entropy); // Resolve powershell.exe. string powerShellExe = HostContext.GetService <IPowerShellExeUtil>().GetPath(); ArgUtil.NotNullOrEmpty(powerShellExe, nameof(powerShellExe)); // Determine whether the script file is rooted. // TODO: If script file begins and ends with a double-quote, trim quotes before making determination. Likewise when determining whether the file exists. bool isScriptFileRooted = false; try { // Path.IsPathRooted throws if illegal characters are in the path. isScriptFileRooted = Path.IsPathRooted(scriptFile); } catch (Exception ex) { Trace.Info($"Unable to determine whether the script file is rooted: {ex.Message}"); } Trace.Info($"Script file is rooted: {isScriptFileRooted}"); // Determine the working directory. string workingDirectory; if (!string.IsNullOrEmpty(Data.WorkingDirectory)) { workingDirectory = Data.WorkingDirectory; } else { if (isScriptFileRooted && File.Exists(scriptFile)) { workingDirectory = Path.GetDirectoryName(scriptFile); } else { workingDirectory = Path.Combine(TaskDirectory, "DefaultTaskWorkingDirectory"); } } ExecutionContext.Debug($"Working directory: '{workingDirectory}'"); Directory.CreateDirectory(workingDirectory); // Invoke the process. ExecutionContext.Debug($"{powerShellExe} {powerShellExeArgs}"); ExecutionContext.Command(nestedExpression); using (var processInvoker = HostContext.CreateService <IProcessInvoker>()) { processInvoker.OutputDataReceived += OnOutputDataReceived; processInvoker.ErrorDataReceived += OnErrorDataReceived; int exitCode = await processInvoker.ExecuteAsync( workingDirectory : workingDirectory, fileName : powerShellExe, arguments : powerShellExeArgs, environment : Environment, requireExitCodeZero : false, outputEncoding : null, killProcessOnCancel : false, cancellationToken : ExecutionContext.CancellationToken); FlushErrorData(); // Fail on error count. if (_failOnStandardError && _errorCount > 0) { if (ExecutionContext.Result != null) { Trace.Info($"Task result already set. Not failing due to error count ({_errorCount})."); } else { throw new Exception(StringUtil.Loc("ProcessCompletedWithCode0Errors1", exitCode, _errorCount)); } } // Fail on non-zero exit code. if (exitCode != 0) { throw new Exception(StringUtil.Loc("ProcessCompletedWithExitCode0", exitCode)); } } } finally { try { if (string.Equals(Data.ScriptType, InlineScriptType, StringComparison.OrdinalIgnoreCase) && !string.IsNullOrEmpty(scriptFile) && File.Exists(scriptFile)) { File.Delete(scriptFile); } } catch (Exception ex) { ExecutionContext.Warning(StringUtil.Loc("FailedToDeleteTempScript", scriptFile, ex.Message)); Trace.Error(ex); } } }
public async Task RunPluginTaskAsync(IExecutionContext context, string plugin, Dictionary <string, string> inputs, Dictionary <string, string> environment, Variables runtimeVariables, EventHandler <ProcessDataReceivedEventArgs> outputHandler) { ArgUtil.NotNullOrEmpty(plugin, nameof(plugin)); // Only allow plugins we defined if (!_taskPlugins.Contains(plugin)) { throw new NotSupportedException(plugin); } // Resolve the working directory. string workingDirectory = HostContext.GetDirectory(WellKnownDirectory.Work); ArgUtil.Directory(workingDirectory, nameof(workingDirectory)); // Agent.PluginHost string file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), $"Agent.PluginHost{Util.IOUtil.ExeExtension}"); ArgUtil.File(file, $"Agent.PluginHost{Util.IOUtil.ExeExtension}"); // Agent.PluginHost's arguments string arguments = $"task \"{plugin}\""; // construct plugin context var target = context.StepTarget(); Variables.TranslationMethod translateToHostPath = Variables.DefaultStringTranslator; ContainerInfo containerInfo = target as ContainerInfo; // Since plugins run on the host, but the inputs and variables have already been translated // to the container path, we need to convert them back to the host path // TODO: look to see if there is a better way to not have translate these back if (containerInfo != null) { var newInputs = new Dictionary <string, string>(); foreach (var entry in inputs) { newInputs[entry.Key] = containerInfo.TranslateToHostPath(entry.Value); } inputs = newInputs; translateToHostPath = (string val) => { return(containerInfo.TranslateToHostPath(val)); }; } AgentTaskPluginExecutionContext pluginContext = new AgentTaskPluginExecutionContext { Inputs = inputs, Repositories = context.Repositories, Endpoints = context.Endpoints, Container = containerInfo, //TODO: Figure out if this needs to have all the containers or just the one for the current step JobSettings = context.JobSettings, }; // variables runtimeVariables.CopyInto(pluginContext.Variables, translateToHostPath); context.TaskVariables.CopyInto(pluginContext.TaskVariables, translateToHostPath); using (var processInvoker = HostContext.CreateService <IProcessInvoker>()) { var redirectStandardIn = new InputQueue <string>(); redirectStandardIn.Enqueue(JsonUtility.ToString(pluginContext)); processInvoker.OutputDataReceived += outputHandler; processInvoker.ErrorDataReceived += outputHandler; // Execute the process. Exit code 0 should always be returned. // A non-zero exit code indicates infrastructural failure. // Task failure should be communicated over STDOUT using ## commands. await processInvoker.ExecuteAsync(workingDirectory : workingDirectory, fileName : file, arguments : arguments, environment : environment, requireExitCodeZero : true, outputEncoding : Encoding.UTF8, killProcessOnCancel : false, redirectStandardIn : redirectStandardIn, cancellationToken : context.CancellationToken); } }
public async Task RunAsync() { // Validate args. Trace.Entering(); ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext)); ArgUtil.NotNull(Action, nameof(Action)); var taskManager = HostContext.GetService <IActionManager>(); var handlerFactory = HostContext.GetService <IHandlerFactory>(); // Load the task definition and choose the handler. Definition definition = taskManager.LoadAction(ExecutionContext, Action); ArgUtil.NotNull(definition, nameof(definition)); ActionExecutionData handlerData = definition.Data?.Execution; ArgUtil.NotNull(handlerData, nameof(handlerData)); List <JobExtensionRunner> localActionContainerSetupSteps = null; // Handle Composite Local Actions // Need to download and expand the tree of referenced actions if (handlerData.ExecutionType == ActionExecutionType.Composite && handlerData is CompositeActionExecutionData compositeHandlerData && Stage == ActionRunStage.Main && Action.Reference is Pipelines.RepositoryPathReference localAction && string.Equals(localAction.RepositoryType, Pipelines.PipelineConstants.SelfAlias, StringComparison.OrdinalIgnoreCase)) { var actionManager = HostContext.GetService <IActionManager>(); var prepareResult = await actionManager.PrepareActionsAsync(ExecutionContext, compositeHandlerData.Steps, ExecutionContext.Id); // Reload definition since post may exist now (from embedded steps that were JIT downloaded) definition = taskManager.LoadAction(ExecutionContext, Action); ArgUtil.NotNull(definition, nameof(definition)); handlerData = definition.Data?.Execution; ArgUtil.NotNull(handlerData, nameof(handlerData)); // Save container setup steps so we can reference them later localActionContainerSetupSteps = prepareResult.ContainerSetupSteps; } if (handlerData.HasPre && Action.Reference is Pipelines.RepositoryPathReference repoAction && string.Equals(repoAction.RepositoryType, Pipelines.PipelineConstants.SelfAlias, StringComparison.OrdinalIgnoreCase)) { ExecutionContext.Warning($"`pre` execution is not supported for local action from '{repoAction.Path}'"); } // The action has post cleanup defined. // we need to create timeline record for them and add them to the step list that StepRunner is using if (handlerData.HasPost && (Stage == ActionRunStage.Pre || Stage == ActionRunStage.Main)) { string postDisplayName = $"Post {this.DisplayName}"; if (Stage == ActionRunStage.Pre && this.DisplayName.StartsWith("Pre ", StringComparison.OrdinalIgnoreCase)) { // Trim the leading `Pre ` from the display name. // Otherwise, we will get `Post Pre xxx` as DisplayName for the Post step. postDisplayName = $"Post {this.DisplayName.Substring("Pre ".Length)}"; } var repositoryReference = Action.Reference as RepositoryPathReference; var pathString = string.IsNullOrEmpty(repositoryReference.Path) ? string.Empty : $"/{repositoryReference.Path}"; var repoString = string.IsNullOrEmpty(repositoryReference.Ref) ? $"{repositoryReference.Name}{pathString}" : $"{repositoryReference.Name}{pathString}@{repositoryReference.Ref}"; ExecutionContext.Debug($"Register post job cleanup for action: {repoString}"); var actionRunner = HostContext.CreateService <IActionRunner>(); actionRunner.Action = Action; actionRunner.Stage = ActionRunStage.Post; actionRunner.Condition = handlerData.CleanupCondition; actionRunner.DisplayName = postDisplayName; ExecutionContext.RegisterPostJobStep(actionRunner); } IStepHost stepHost = HostContext.CreateService <IDefaultStepHost>(); // Makes directory for event_path data var tempDirectory = HostContext.GetDirectory(WellKnownDirectory.Temp); var workflowDirectory = Path.Combine(tempDirectory, "_github_workflow"); Directory.CreateDirectory(workflowDirectory); var gitHubEvent = ExecutionContext.GetGitHubContext("event"); // adds the GitHub event path/file if the event exists if (gitHubEvent != null) { var workflowFile = Path.Combine(workflowDirectory, "event.json"); Trace.Info($"Write event payload to {workflowFile}"); File.WriteAllText(workflowFile, gitHubEvent, new UTF8Encoding(false)); ExecutionContext.SetGitHubContext("event_path", workflowFile); } // Set GITHUB_ACTION_REPOSITORY if this Action is from a repository if (Action.Reference is Pipelines.RepositoryPathReference repoPathReferenceAction && !string.Equals(repoPathReferenceAction.RepositoryType, Pipelines.PipelineConstants.SelfAlias, StringComparison.OrdinalIgnoreCase)) { ExecutionContext.SetGitHubContext("action_repository", repoPathReferenceAction.Name); ExecutionContext.SetGitHubContext("action_ref", repoPathReferenceAction.Ref); }
public IHandler Create( IExecutionContext executionContext, Pipelines.ActionStepDefinitionReference action, IStepHost stepHost, ActionExecutionData data, Dictionary <string, string> inputs, Dictionary <string, string> environment, Variables runtimeVariables, string actionDirectory, List <JobExtensionRunner> localActionContainerSetupSteps) { // Validate args. Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(stepHost, nameof(stepHost)); ArgUtil.NotNull(data, nameof(data)); ArgUtil.NotNull(inputs, nameof(inputs)); ArgUtil.NotNull(environment, nameof(environment)); ArgUtil.NotNull(runtimeVariables, nameof(runtimeVariables)); // Create the handler. IHandler handler; if (data.ExecutionType == ActionExecutionType.Container) { handler = HostContext.CreateService <IContainerActionHandler>(); (handler as IContainerActionHandler).Data = data as ContainerActionExecutionData; } else if (data.ExecutionType == ActionExecutionType.NodeJS) { handler = HostContext.CreateService <INodeScriptActionHandler>(); (handler as INodeScriptActionHandler).Data = data as NodeJSActionExecutionData; } else if (data.ExecutionType == ActionExecutionType.Script) { handler = HostContext.CreateService <IScriptHandler>(); (handler as IScriptHandler).Data = data as ScriptActionExecutionData; } else if (data.ExecutionType == ActionExecutionType.Plugin) { // Runner plugin handler = HostContext.CreateService <IRunnerPluginHandler>(); (handler as IRunnerPluginHandler).Data = data as PluginActionExecutionData; } else if (data.ExecutionType == ActionExecutionType.Composite) { handler = HostContext.CreateService <ICompositeActionHandler>(); (handler as ICompositeActionHandler).Data = data as CompositeActionExecutionData; } else { // This should never happen. throw new NotSupportedException(data.ExecutionType.ToString()); } handler.Action = action; handler.Environment = environment; handler.RuntimeVariables = runtimeVariables; handler.ExecutionContext = executionContext; handler.StepHost = stepHost; handler.Inputs = inputs; handler.ActionDirectory = actionDirectory; handler.LocalActionContainerSetupSteps = localActionContainerSetupSteps; return(handler); }
public void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token) { // Validation Trace.Entering(); ArgUtil.NotNull(message, nameof(message)); ArgUtil.NotNull(message.Resources, nameof(message.Resources)); ArgUtil.NotNull(message.Variables, nameof(message.Variables)); ArgUtil.NotNull(message.Plan, nameof(message.Plan)); _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token); // Features Features = PlanUtil.GetFeatures(message.Plan); // Endpoints Endpoints = message.Resources.Endpoints; // Variables Variables = new Variables(HostContext, message.Variables); // Environment variables shared across all actions EnvironmentVariables = new Dictionary <string, string>(VarUtil.EnvironmentVariableKeyComparer); // Service container info ServiceContainers = new List <ContainerInfo>(); // Steps context (StepsRunner manages adding the scoped steps context) StepsContext = new StepsContext(); // Scopes Scopes = new Dictionary <String, ContextScope>(StringComparer.OrdinalIgnoreCase); if (message.Scopes?.Count > 0) { foreach (var scope in message.Scopes) { Scopes[scope.Name] = scope; } } // File table FileTable = new List <String>(message.FileTable ?? new string[0]); // Expression functions if (Variables.GetBoolean("System.HashFilesV2") == true) { ExpressionConstants.UpdateFunction <Handlers.HashFiles>("hashFiles", 1, byte.MaxValue); } // Expression values if (message.ContextData?.Count > 0) { foreach (var pair in message.ContextData) { ExpressionValues[pair.Key] = pair.Value; } } ExpressionValues["secrets"] = Variables.ToSecretsContext(); ExpressionValues["runner"] = new RunnerContext(); ExpressionValues["job"] = new JobContext(); Trace.Info("Initialize GitHub context"); var githubAccessToken = new StringContextData(Variables.Get("system.github.token")); var base64EncodedToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{githubAccessToken}")); HostContext.SecretMasker.AddValue(base64EncodedToken); var githubContext = new GitHubContext(); githubContext["token"] = githubAccessToken; var githubDictionary = ExpressionValues["github"].AssertDictionary("github"); foreach (var pair in githubDictionary) { githubContext[pair.Key] = pair.Value; } ExpressionValues["github"] = githubContext; Trace.Info("Initialize Env context"); #if OS_WINDOWS ExpressionValues["env"] = new DictionaryContextData(); #else ExpressionValues["env"] = new CaseSensitiveDictionaryContextData(); #endif // Prepend Path PrependPath = new List <string>(); // JobSteps for job ExecutionContext JobSteps = new Queue <IStep>(); // PostJobSteps for job ExecutionContext PostJobSteps = new Stack <IStep>(); // Job timeline record. InitializeTimelineRecord( timelineId: message.Timeline.Id, timelineRecordId: message.JobId, parentTimelineRecordId: null, recordType: ExecutionContextType.Job, displayName: message.JobDisplayName, refName: message.JobName, order: null); // The job timeline record's order is set by server. // Logger (must be initialized before writing warnings). _logger = HostContext.CreateService <IPagingLogger>(); _logger.Setup(_mainTimelineId, _record.Id); // Initialize 'echo on action command success' property, default to false, unless Step_Debug is set EchoOnActionCommand = Variables.Step_Debug ?? false; // Verbosity (from GitHub.Step_Debug). WriteDebug = Variables.Step_Debug ?? false; // Hook up JobServerQueueThrottling event, we will log warning on server tarpit. _jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived; }
private async Task RunAsync(AgentJobRequestMessage message, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken) { if (previousJobDispatch != null) { Trace.Verbose($"Make sure the previous job request {previousJobDispatch.JobId} has successfully finished on worker."); await EnsureDispatchFinished(previousJobDispatch); } else { Trace.Verbose($"This is the first job request."); } var term = HostContext.GetService <ITerminal>(); term.WriteLine(StringUtil.Loc("RunningJob", DateTime.UtcNow, message.JobName)); // first job request renew succeed. TaskCompletionSource <int> firstJobRequestRenewed = new TaskCompletionSource <int>(); var notification = HostContext.GetService <IJobNotification>(); // lock renew cancellation token. using (var lockRenewalTokenSource = new CancellationTokenSource()) using (var workerProcessCancelTokenSource = new CancellationTokenSource()) { long requestId = message.RequestId; Guid lockToken = message.LockToken; // start renew job request Trace.Info("Start renew job request."); Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, firstJobRequestRenewed, lockRenewalTokenSource.Token); // wait till first renew succeed or job request is canceled // not even start worker if the first renew fail await Task.WhenAny(firstJobRequestRenewed.Task, renewJobRequest, Task.Delay(-1, jobRequestCancellationToken)); if (renewJobRequest.IsCompleted) { // renew job request task complete means we run out of retry for the first job request renew. // TODO: not need to return anything. Trace.Info("Unable to renew job request for the first time, stop dispatching job to worker."); return; } if (jobRequestCancellationToken.IsCancellationRequested) { await CompleteJobRequestAsync(_poolId, message, lockToken, TaskResult.Canceled); return; } // we get first jobrequest renew succeed. // send notification to machine provisioner. await notification.JobStarted(message.JobId); Task <int> workerProcessTask = null; using (var processChannel = HostContext.CreateService <IProcessChannel>()) using (var processInvoker = HostContext.CreateService <IProcessInvoker>()) { // Start the process channel. // It's OK if StartServer bubbles an execption after the worker process has already started. // The worker will shutdown after 30 seconds if it hasn't received the job message. processChannel.StartServer( // Delegate to start the child process. startProcess: (string pipeHandleOut, string pipeHandleIn) => { // Validate args. ArgUtil.NotNullOrEmpty(pipeHandleOut, nameof(pipeHandleOut)); ArgUtil.NotNullOrEmpty(pipeHandleIn, nameof(pipeHandleIn)); // Start the child process. var assemblyDirectory = IOUtil.GetBinPath(); string workerFileName = Path.Combine(assemblyDirectory, _workerProcessName); workerProcessTask = processInvoker.ExecuteAsync( workingDirectory: assemblyDirectory, fileName: workerFileName, arguments: "spawnclient " + pipeHandleOut + " " + pipeHandleIn, environment: null, cancellationToken: workerProcessCancelTokenSource.Token); }); // Send the job request message. // Kill the worker process if sending the job message times out. The worker // process may have successfully received the job message. try { Trace.Info("Send job request message to worker."); using (var csSendJobRequest = new CancellationTokenSource(ChannelTimeout)) { await processChannel.SendAsync( messageType : MessageType.NewJobRequest, body : JsonUtility.ToString(message), cancellationToken : csSendJobRequest.Token); } } catch (OperationCanceledException) { // message send been cancelled. // timeout 45 sec. kill worker. Trace.Info("Job request message sending been cancelled, kill running worker."); workerProcessCancelTokenSource.Cancel(); try { await workerProcessTask; } catch (OperationCanceledException) { // worker process been killed. } Trace.Info("Stop renew job request."); // stop renew lock lockRenewalTokenSource.Cancel(); // renew job request should never blows up. await renewJobRequest; // not finish the job request since the job haven't run on worker at all, we will not going to set a result to server. return; } TaskResult resultOnAbandonOrCancel = TaskResult.Succeeded; // wait for renewlock, worker process or cancellation token been fired. var completedTask = await Task.WhenAny(renewJobRequest, workerProcessTask, Task.Delay(-1, jobRequestCancellationToken)); if (completedTask == workerProcessTask) { // worker finished successfully, complete job request with result, stop renew lock, job has finished. int returnCode = await workerProcessTask; Trace.Info("Worker finished. Code: " + returnCode); TaskResult result = TaskResultUtil.TranslateFromReturnCode(returnCode); Trace.Info($"finish job request with result: {result}"); term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobName, result)); // complete job request await CompleteJobRequestAsync(_poolId, message, lockToken, result); Trace.Info("Stop renew job request."); // stop renew lock lockRenewalTokenSource.Cancel(); // renew job request should never blows up. await renewJobRequest; return; } else if (completedTask == renewJobRequest) { resultOnAbandonOrCancel = TaskResult.Abandoned; } else { resultOnAbandonOrCancel = TaskResult.Canceled; } // renew job request completed or job request cancellation token been fired for RunAsync(jobrequestmessage) // cancel worker gracefully first, then kill it after 45 sec try { Trace.Info("Send job cancellation message to worker."); using (var csSendCancel = new CancellationTokenSource(ChannelTimeout)) { await processChannel.SendAsync( messageType : MessageType.CancelRequest, body : string.Empty, cancellationToken : csSendCancel.Token); } } catch (OperationCanceledException) { // message send been cancelled. Trace.Info("Job cancel message sending been cancelled, kill running worker."); workerProcessCancelTokenSource.Cancel(); try { await workerProcessTask; } catch (OperationCanceledException) { // worker process been killed. } } // wait worker to exit within 45 sec, then kill worker. using (var csKillWorker = new CancellationTokenSource(TimeSpan.FromSeconds(45))) { completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, csKillWorker.Token)); } // worker haven't exit within 45 sec. if (completedTask != workerProcessTask) { Trace.Info("worker process haven't exit after 45 sec, kill running worker."); workerProcessCancelTokenSource.Cancel(); try { await workerProcessTask; } catch (OperationCanceledException) { // worker process been killed. } } Trace.Info($"finish job request with result: {resultOnAbandonOrCancel}"); term.WriteLine(StringUtil.Loc("JobCompleted", DateTime.UtcNow, message.JobName, resultOnAbandonOrCancel)); // complete job request with cancel result, stop renew lock, job has finished. //TODO: don't finish job request on abandon await CompleteJobRequestAsync(_poolId, message, lockToken, resultOnAbandonOrCancel); Trace.Info("Stop renew job request."); // stop renew lock lockRenewalTokenSource.Cancel(); // renew job request should never blows up. await renewJobRequest; } } }
public async Task <Boolean> VerifyAsync(Definition definition, CancellationToken token) { ArgUtil.NotNull(definition, nameof(definition)); // This is used for the Checkout task. // We can consider it verified since it's embedded in the Agent code. if (String.IsNullOrEmpty(definition.ZipPath)) { return(true); } // Find NuGet String nugetPath = WhichUtil.Which("nuget", require: true); var configurationStore = HostContext.GetService <IConfigurationStore>(); AgentSettings settings = configurationStore.GetSettings(); SignatureVerificationSettings verificationSettings = settings.SignatureVerification; String taskZipPath = definition.ZipPath; String taskNugetPath = definition.ZipPath.Replace(".zip", ".nupkg"); // Rename .zip to .nupkg File.Move(taskZipPath, taskNugetPath); String arguments = $"verify -Signatures \"{taskNugetPath}\" -Verbosity Detailed"; if (verificationSettings?.Fingerprints != null && verificationSettings.Fingerprints.Count > 0) { String fingerprint = String.Join(";", verificationSettings.Fingerprints); arguments += $" -CertificateFingerprint \"{fingerprint}\""; } Trace.Info($"nuget arguments: {arguments}"); // Run nuget verify using (var processInvoker = HostContext.CreateService <IProcessInvoker>()) { processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs args) => { if (!string.IsNullOrEmpty(args.Data)) { Trace.Info(args.Data); } }; int exitCode = await processInvoker.ExecuteAsync(workingDirectory : HostContext.GetDirectory(WellKnownDirectory.Root), fileName : nugetPath, arguments : arguments, environment : null, requireExitCodeZero : false, outputEncoding : null, killProcessOnCancel : false, cancellationToken : token); // Rename back to zip File.Move(taskNugetPath, taskZipPath); if (exitCode != 0) { return(false); } } return(true); }
public void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token) { // Validation Trace.Entering(); ArgUtil.NotNull(message, nameof(message)); ArgUtil.NotNull(message.Resources, nameof(message.Resources)); ArgUtil.NotNull(message.Variables, nameof(message.Variables)); ArgUtil.NotNull(message.Plan, nameof(message.Plan)); _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token); // Features Features = PlanUtil.GetFeatures(message.Plan); // Endpoints Endpoints = message.Resources.Endpoints; // SecureFiles SecureFiles = message.Resources.SecureFiles; // Repositories Repositories = message.Resources.Repositories; // Variables (constructor performs initial recursive expansion) List <string> warnings; Variables = new Variables(HostContext, message.Variables, out warnings); // Prepend Path PrependPath = new List <string>(); // Docker (JobContainer) string imageName = Variables.Get("_PREVIEW_VSTS_DOCKER_IMAGE"); if (string.IsNullOrEmpty(imageName)) { imageName = Environment.GetEnvironmentVariable("_PREVIEW_VSTS_DOCKER_IMAGE"); } if (!string.IsNullOrEmpty(imageName) && string.IsNullOrEmpty(message.JobContainer)) { var dockerContainer = new Pipelines.ContainerResource() { Alias = "vsts_container_preview" }; dockerContainer.Properties.Set("image", imageName); Container = new ContainerInfo(HostContext, dockerContainer); } else if (!string.IsNullOrEmpty(message.JobContainer)) { Container = new ContainerInfo(HostContext, message.Resources.Containers.Single(x => string.Equals(x.Alias, message.JobContainer, StringComparison.OrdinalIgnoreCase))); } else { Container = null; } // Docker (Sidecar Containers) SidecarContainers = new List <ContainerInfo>(); foreach (var sidecar in message.JobSidecarContainers) { var networkAlias = sidecar.Key; var containerResourceAlias = sidecar.Value; var containerResource = message.Resources.Containers.Single(c => string.Equals(c.Alias, containerResourceAlias, StringComparison.OrdinalIgnoreCase)); SidecarContainers.Add(new ContainerInfo(HostContext, containerResource, isJobContainer: false) { ContainerNetworkAlias = networkAlias }); } // Proxy variables var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>(); if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress)) { Variables.Set(Constants.Variables.Agent.ProxyUrl, agentWebProxy.ProxyAddress); Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY", string.Empty); if (!string.IsNullOrEmpty(agentWebProxy.ProxyUsername)) { Variables.Set(Constants.Variables.Agent.ProxyUsername, agentWebProxy.ProxyUsername); Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_USERNAME", string.Empty); } if (!string.IsNullOrEmpty(agentWebProxy.ProxyPassword)) { Variables.Set(Constants.Variables.Agent.ProxyPassword, agentWebProxy.ProxyPassword, true); Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_PASSWORD", string.Empty); } if (agentWebProxy.ProxyBypassList.Count > 0) { Variables.Set(Constants.Variables.Agent.ProxyBypassList, JsonUtility.ToString(agentWebProxy.ProxyBypassList)); } } // Certificate variables var agentCert = HostContext.GetService <IAgentCertificateManager>(); if (agentCert.SkipServerCertificateValidation) { Variables.Set(Constants.Variables.Agent.SslSkipCertValidation, bool.TrueString); } if (!string.IsNullOrEmpty(agentCert.CACertificateFile)) { Variables.Set(Constants.Variables.Agent.SslCAInfo, agentCert.CACertificateFile); } if (!string.IsNullOrEmpty(agentCert.ClientCertificateFile) && !string.IsNullOrEmpty(agentCert.ClientCertificatePrivateKeyFile) && !string.IsNullOrEmpty(agentCert.ClientCertificateArchiveFile)) { Variables.Set(Constants.Variables.Agent.SslClientCert, agentCert.ClientCertificateFile); Variables.Set(Constants.Variables.Agent.SslClientCertKey, agentCert.ClientCertificatePrivateKeyFile); Variables.Set(Constants.Variables.Agent.SslClientCertArchive, agentCert.ClientCertificateArchiveFile); if (!string.IsNullOrEmpty(agentCert.ClientCertificatePassword)) { Variables.Set(Constants.Variables.Agent.SslClientCertPassword, agentCert.ClientCertificatePassword, true); } } // Runtime option variables var runtimeOptions = HostContext.GetService <IConfigurationStore>().GetAgentRuntimeOptions(); if (runtimeOptions != null) { #if OS_WINDOWS if (runtimeOptions.GitUseSecureChannel) { Variables.Set(Constants.Variables.Agent.GitUseSChannel, runtimeOptions.GitUseSecureChannel.ToString()); } #endif } // Job timeline record. InitializeTimelineRecord( timelineId: message.Timeline.Id, timelineRecordId: message.JobId, parentTimelineRecordId: null, recordType: ExecutionContextType.Job, displayName: message.JobDisplayName, refName: message.JobName, order: null); // The job timeline record's order is set by server. // Logger (must be initialized before writing warnings). _logger = HostContext.CreateService <IPagingLogger>(); _logger.Setup(_mainTimelineId, _record.Id); // Log warnings from recursive variable expansion. warnings?.ForEach(x => this.Warning(x)); // Verbosity (from system.debug). WriteDebug = Variables.System_Debug ?? false; // Hook up JobServerQueueThrottling event, we will log warning on server tarpit. _jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived; }
/// <summary> /// _work /// \_update /// \bin /// \externals /// \run.sh /// \run.cmd /// \package.zip //temp download .zip/.tar.gz /// </summary> /// <param name="token"></param> /// <returns></returns> private async Task DownloadLatestAgent(CancellationToken token) { string latestAgentDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), Constants.Path.UpdateDirectory); IOUtil.DeleteDirectory(latestAgentDirectory, token); Directory.CreateDirectory(latestAgentDirectory); int agentSuffix = 1; string archiveFile = null; bool downloadSucceeded = false; try { // Download the agent, using multiple attempts in order to be resilient against any networking/CDN issues for (int attempt = 1; attempt <= Constants.AgentDownloadRetryMaxAttempts; attempt++) { // Generate an available package name, and do our best effort to clean up stale local zip files while (true) { if (_targetPackage.Platform.StartsWith("win")) { archiveFile = Path.Combine(latestAgentDirectory, $"agent{agentSuffix}.zip"); } else { archiveFile = Path.Combine(latestAgentDirectory, $"agent{agentSuffix}.tar.gz"); } try { // delete .zip file if (!string.IsNullOrEmpty(archiveFile) && File.Exists(archiveFile)) { Trace.Verbose("Deleting latest agent package zip '{0}'", archiveFile); IOUtil.DeleteFile(archiveFile); } break; } catch (Exception ex) { // couldn't delete the file for whatever reason, so generate another name Trace.Warning("Failed to delete agent package zip '{0}'. Exception: {1}", archiveFile, ex); agentSuffix++; } } // Allow a 15-minute package download timeout, which is good enough to update the agent from a 1 Mbit/s ADSL connection. if (!int.TryParse(Environment.GetEnvironmentVariable("AZP_AGENT_DOWNLOAD_TIMEOUT") ?? string.Empty, out int timeoutSeconds)) { timeoutSeconds = 15 * 60; } Trace.Info($"Attempt {attempt}: save latest agent into {archiveFile}."); using (var downloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds))) using (var downloadCts = CancellationTokenSource.CreateLinkedTokenSource(downloadTimeout.Token, token)) { try { Trace.Info($"Download agent: begin download"); //open zip stream in async mode using (var handler = HostContext.CreateHttpClientHandler()) using (var httpClient = new HttpClient(handler)) using (var fs = new FileStream(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 4096, useAsync: true)) using (var result = await httpClient.GetStreamAsync(_targetPackage.DownloadUrl)) { //81920 is the default used by System.IO.Stream.CopyTo and is under the large object heap threshold (85k). await result.CopyToAsync(fs, 81920, downloadCts.Token); await fs.FlushAsync(downloadCts.Token); } Trace.Info($"Download agent: finished download"); downloadSucceeded = true; break; } catch (OperationCanceledException) when(token.IsCancellationRequested) { Trace.Info($"Agent download has been canceled."); throw; } catch (Exception ex) { if (downloadCts.Token.IsCancellationRequested) { Trace.Warning($"Agent download has timed out after {timeoutSeconds} seconds"); } Trace.Warning($"Failed to get package '{archiveFile}' from '{_targetPackage.DownloadUrl}'. Exception {ex}"); } } } if (!downloadSucceeded) { throw new TaskCanceledException($"Agent package '{archiveFile}' failed after {Constants.AgentDownloadRetryMaxAttempts} download attempts"); } // If we got this far, we know that we've successfully downloadeded the agent package if (archiveFile.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) { ZipFile.ExtractToDirectory(archiveFile, latestAgentDirectory); } else if (archiveFile.EndsWith(".tar.gz", StringComparison.OrdinalIgnoreCase)) { string tar = WhichUtil.Which("tar", trace: Trace); if (string.IsNullOrEmpty(tar)) { throw new NotSupportedException($"tar -xzf"); } // tar -xzf using (var processInvoker = HostContext.CreateService <IProcessInvoker>()) { processInvoker.OutputDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) => { if (!string.IsNullOrEmpty(args.Data)) { Trace.Info(args.Data); } }); processInvoker.ErrorDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) => { if (!string.IsNullOrEmpty(args.Data)) { Trace.Error(args.Data); } }); int exitCode = await processInvoker.ExecuteAsync(latestAgentDirectory, tar, $"-xzf \"{archiveFile}\"", null, token); if (exitCode != 0) { throw new NotSupportedException($"Can't use 'tar -xzf' extract archive file: {archiveFile}. return code: {exitCode}."); } } } else { throw new NotSupportedException($"{archiveFile}"); } Trace.Info($"Finished getting latest agent package at: {latestAgentDirectory}."); } finally { try { // delete .zip file if (!string.IsNullOrEmpty(archiveFile) && File.Exists(archiveFile)) { Trace.Verbose("Deleting latest agent package zip: {0}", archiveFile); IOUtil.DeleteFile(archiveFile); } } catch (Exception ex) { //it is not critical if we fail to delete the .zip file Trace.Warning("Failed to delete agent package zip '{0}'. Exception: {1}", archiveFile, ex); } } // copy latest agent into agent root folder // copy bin from _work/_update -> bin.version under root string binVersionDir = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"{Constants.Path.BinDirectory}.{_targetPackage.Version}"); Directory.CreateDirectory(binVersionDir); Trace.Info($"Copy {Path.Combine(latestAgentDirectory, Constants.Path.BinDirectory)} to {binVersionDir}."); IOUtil.CopyDirectory(Path.Combine(latestAgentDirectory, Constants.Path.BinDirectory), binVersionDir, token); // copy externals from _work/_update -> externals.version under root string externalsVersionDir = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"{Constants.Path.ExternalsDirectory}.{_targetPackage.Version}"); Directory.CreateDirectory(externalsVersionDir); Trace.Info($"Copy {Path.Combine(latestAgentDirectory, Constants.Path.ExternalsDirectory)} to {externalsVersionDir}."); IOUtil.CopyDirectory(Path.Combine(latestAgentDirectory, Constants.Path.ExternalsDirectory), externalsVersionDir, token); // copy and replace all .sh/.cmd files Trace.Info($"Copy any remaining .sh/.cmd files into agent root."); foreach (FileInfo file in new DirectoryInfo(latestAgentDirectory).GetFiles() ?? new FileInfo[0]) { // Copy and replace the file. file.CopyTo(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), file.Name), true); } // for windows service back compat with old windows agent, we need make sure the servicehost.exe is still the old name // if the current bin folder has VsoAgentService.exe, then the new agent bin folder needs VsoAgentService.exe as well if (PlatformUtil.RunningOnWindows) { if (File.Exists(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), "VsoAgentService.exe"))) { Trace.Info($"Make a copy of AgentService.exe, name it VsoAgentService.exe"); File.Copy(Path.Combine(binVersionDir, "AgentService.exe"), Path.Combine(binVersionDir, "VsoAgentService.exe"), true); File.Copy(Path.Combine(binVersionDir, "AgentService.exe.config"), Path.Combine(binVersionDir, "VsoAgentService.exe.config"), true); Trace.Info($"Make a copy of Agent.Listener.exe, name it VsoAgent.exe"); File.Copy(Path.Combine(binVersionDir, "Agent.Listener.exe"), Path.Combine(binVersionDir, "VsoAgent.exe"), true); File.Copy(Path.Combine(binVersionDir, "Agent.Listener.dll"), Path.Combine(binVersionDir, "VsoAgent.dll"), true); // in case of we remove all pdb file from agent package. if (File.Exists(Path.Combine(binVersionDir, "AgentService.pdb"))) { File.Copy(Path.Combine(binVersionDir, "AgentService.pdb"), Path.Combine(binVersionDir, "VsoAgentService.pdb"), true); } if (File.Exists(Path.Combine(binVersionDir, "Agent.Listener.pdb"))) { File.Copy(Path.Combine(binVersionDir, "Agent.Listener.pdb"), Path.Combine(binVersionDir, "VsoAgent.pdb"), true); } } } }
private async Task RunStepAsync(IStep step, CancellationToken jobCancellationToken) { // Check to see if we can expand the display name if (step is IActionRunner actionRunner && actionRunner.Stage == ActionRunStage.Main && actionRunner.TryEvaluateDisplayName(step.ExecutionContext.ExpressionValues, step.ExecutionContext)) { step.ExecutionContext.UpdateTimelineRecordDisplayName(actionRunner.DisplayName); } // Start the step. Trace.Info("Starting the step."); step.ExecutionContext.Debug($"Starting: {step.DisplayName}"); // Set the timeout var timeoutMinutes = 0; var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(); try { timeoutMinutes = templateEvaluator.EvaluateStepTimeout(step.Timeout, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions); } catch (Exception ex) { Trace.Info("An error occurred when attempting to determine the step timeout."); Trace.Error(ex); step.ExecutionContext.Error("An error occurred when attempting to determine the step timeout."); step.ExecutionContext.Error(ex); } if (timeoutMinutes > 0) { var timeout = TimeSpan.FromMinutes(timeoutMinutes); step.ExecutionContext.SetTimeout(timeout); } #if OS_WINDOWS try { if (Console.InputEncoding.CodePage != 65001) { using (var p = HostContext.CreateService <IProcessInvoker>()) { // Use UTF8 code page int exitCode = await p.ExecuteAsync(workingDirectory : HostContext.GetDirectory(WellKnownDirectory.Work), fileName : WhichUtil.Which("chcp", true, Trace), arguments : "65001", environment : null, requireExitCodeZero : false, outputEncoding : null, killProcessOnCancel : false, redirectStandardIn : null, inheritConsoleHandler : true, cancellationToken : step.ExecutionContext.CancellationToken); if (exitCode == 0) { Trace.Info("Successfully returned to code page 65001 (UTF8)"); } else { Trace.Warning($"'chcp 65001' failed with exit code {exitCode}"); } } } } catch (Exception ex) { Trace.Warning($"'chcp 65001' failed with exception {ex.Message}"); } #endif try { await step.RunAsync(); } catch (OperationCanceledException ex) { if (step.ExecutionContext.CancellationToken.IsCancellationRequested && !jobCancellationToken.IsCancellationRequested) { Trace.Error($"Caught timeout exception from step: {ex.Message}"); step.ExecutionContext.Error("The action has timed out."); step.ExecutionContext.Result = TaskResult.Failed; } else { // Log the exception and cancel the step. Trace.Error($"Caught cancellation exception from step: {ex}"); step.ExecutionContext.Error(ex); step.ExecutionContext.Result = TaskResult.Canceled; } } catch (Exception ex) { // Log the error and fail the step. Trace.Error($"Caught exception from step: {ex}"); step.ExecutionContext.Error(ex); step.ExecutionContext.Result = TaskResult.Failed; } // Merge execution context result with command result if (step.ExecutionContext.CommandResult != null) { step.ExecutionContext.Result = TaskResultUtil.MergeTaskResults(step.ExecutionContext.Result, step.ExecutionContext.CommandResult.Value); } // Fixup the step result if ContinueOnError. if (step.ExecutionContext.Result == TaskResult.Failed) { var continueOnError = false; try { continueOnError = templateEvaluator.EvaluateStepContinueOnError(step.ContinueOnError, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions); } catch (Exception ex) { Trace.Info("The step failed and an error occurred when attempting to determine whether to continue on error."); Trace.Error(ex); step.ExecutionContext.Error("The step failed and an error occurred when attempting to determine whether to continue on error."); step.ExecutionContext.Error(ex); } if (continueOnError) { step.ExecutionContext.Outcome = step.ExecutionContext.Result; step.ExecutionContext.Result = TaskResult.Succeeded; Trace.Info($"Updated step result (continue on error)"); } } Trace.Info($"Step result: {step.ExecutionContext.Result}"); // Complete the step context. step.ExecutionContext.Debug($"Finishing: {step.DisplayName}"); }
public async Task RunAsync() { // Validate args. Trace.Entering(); ArgUtil.NotNull(Data, nameof(Data)); ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext)); ArgUtil.NotNull(Inputs, nameof(Inputs)); ArgUtil.Directory(TaskDirectory, nameof(TaskDirectory)); // Resolve the target script. ArgUtil.NotNullOrEmpty(Data.Target, nameof(Data.Target)); string scriptFile = Path.Combine(TaskDirectory, Data.Target); ArgUtil.File(scriptFile, nameof(scriptFile)); // Determine the working directory. string workingDirectory = Data.WorkingDirectory; if (String.IsNullOrEmpty(workingDirectory)) { workingDirectory = Path.GetDirectoryName(scriptFile); } else { if (!Directory.Exists(workingDirectory)) { Directory.CreateDirectory(workingDirectory); } } // scriptName AddEnvironmentVariable("VSTSPSHOSTSCRIPTNAME", scriptFile); // workingFolder AddEnvironmentVariable("VSTSPSHOSTWORKINGFOLDER", workingDirectory); // outputPreference AddEnvironmentVariable("VSTSPSHOSTOUTPUTPREFER", ExecutionContext.WriteDebug ? "Continue" : "SilentlyContinue"); // inputParameters if (Inputs.Count > 0) { AddEnvironmentVariable("VSTSPSHOSTINPUTPARAMETER", JsonUtility.ToString(Inputs)); } List <String> arguments = new List <string>(); Dictionary <String, String> argumentParameters = new Dictionary <String, String>(); if (string.IsNullOrEmpty(Data.ArgumentFormat)) { // treatInputsAsArguments AddEnvironmentVariable("VSTSPSHOSTINPUTISARG", "True"); } else { MatchCollection matches = _argumentMatching.Matches(Data.ArgumentFormat); if (matches[0].Value.StartsWith("-")) { String currentKey = String.Empty; foreach (Match match in matches) { if (match.Value.StartsWith("-")) { currentKey = match.Value.Trim('-'); argumentParameters.Add(currentKey, String.Empty); } else if (!match.Value.StartsWith("-") && !String.IsNullOrEmpty(currentKey)) { argumentParameters[currentKey] = match.Value; currentKey = String.Empty; } else { throw new Exception($"Found value {match.Value} with no corresponding named parameter"); } } } else { foreach (Match match in matches) { arguments.Add(match.Value); } } // arguments if (arguments.Count > 0) { AddEnvironmentVariable("VSTSPSHOSTARGS", JsonUtility.ToString(arguments)); } // argumentParameters if (argumentParameters.Count > 0) { AddEnvironmentVariable("VSTSPSHOSTARGPARAMETER", JsonUtility.ToString(argumentParameters)); } } // push all variable. foreach (var variable in ExecutionContext.Variables.Public.Concat(ExecutionContext.Variables.Private)) { AddEnvironmentVariable("VSTSPSHOSTVAR_" + variable.Key, variable.Value); } // push all public variable. foreach (var variable in ExecutionContext.Variables.Public) { AddEnvironmentVariable("VSTSPSHOSTPUBVAR_" + variable.Key, variable.Value); } // push all endpoints List <String> ids = new List <string>(); foreach (ServiceEndpoint endpoint in ExecutionContext.Endpoints) { string partialKey = null; if (string.Equals(endpoint.Name, ServiceEndpoints.SystemVssConnection, StringComparison.OrdinalIgnoreCase)) { partialKey = ServiceEndpoints.SystemVssConnection.ToUpperInvariant(); AddEnvironmentVariable("VSTSPSHOSTSYSTEMENDPOINT_URL", endpoint.Url.ToString()); AddEnvironmentVariable("VSTSPSHOSTSYSTEMENDPOINT_AUTH", JsonUtility.ToString(endpoint.Authorization)); } else { if (endpoint.Id == Guid.Empty && endpoint.Data.ContainsKey("repositoryId")) { partialKey = endpoint.Data["repositoryId"].ToUpperInvariant(); } else { partialKey = endpoint.Id.ToString("D").ToUpperInvariant(); } ids.Add(partialKey); AddEnvironmentVariable("VSTSPSHOSTENDPOINT_URL_" + partialKey, endpoint.Url.ToString()); AddEnvironmentVariable("VSTSPSHOSTENDPOINT_NAME_" + partialKey, endpoint.Name); AddEnvironmentVariable("VSTSPSHOSTENDPOINT_TYPE_" + partialKey, endpoint.Type); AddEnvironmentVariable("VSTSPSHOSTENDPOINT_AUTH_" + partialKey, JsonUtility.ToString(endpoint.Authorization)); AddEnvironmentVariable("VSTSPSHOSTENDPOINT_DATA_" + partialKey, JsonUtility.ToString(endpoint.Data)); } } if (ids.Count > 0) { AddEnvironmentVariable("VSTSPSHOSTENDPOINT_IDS", JsonUtility.ToString(ids)); } // Invoke the process. using (var processInvoker = HostContext.CreateService <IProcessInvoker>()) { processInvoker.OutputDataReceived += OnDataReceived; processInvoker.ErrorDataReceived += OnDataReceived; try { String vstsPSHostExe = Path.Combine(IOUtil.GetExternalsPath(), "vstshost", "LegacyVSTSPowerShellHost.exe"); Int32 exitCode = await processInvoker.ExecuteAsync(workingDirectory : workingDirectory, fileName : vstsPSHostExe, arguments : "", environment : Environment, cancellationToken : ExecutionContext.CancellationToken); // the exit code from vstsPSHost.exe indicate how many error record we get during execution // -1 exit code means infrastructure failure of Host itself. // this is to match current handler's logic. if (exitCode > 0) { if (ExecutionContext.Result != null) { ExecutionContext.Debug($"Task result already set. Not failing due to error count ({exitCode})."); } else { // We fail task and add issue. ExecutionContext.Result = TaskResult.Failed; ExecutionContext.Error(StringUtil.Loc("PSScriptError", exitCode)); } } else if (exitCode < 0) { // We fail task and add issue. ExecutionContext.Result = TaskResult.Failed; ExecutionContext.Error(StringUtil.Loc("VSTSHostNonZeroReturn", exitCode)); } } finally { processInvoker.OutputDataReceived -= OnDataReceived; processInvoker.ErrorDataReceived -= OnDataReceived; } } }