// log an error issue to job level timeline record private async Task LogWorkerProcessUnhandledException(Pipelines.AgentJobRequestMessage message, string errorMessage) { try { var systemConnection = message.Resources.Endpoints.SingleOrDefault(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection)); ArgUtil.NotNull(systemConnection, nameof(systemConnection)); var jobServer = HostContext.GetService <IJobServer>(); VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection); VssConnection jobConnection = VssUtil.CreateConnection(systemConnection.Url, jobServerCredential); /* Below is the legacy 'OnPremises' code that is currently unused by the runner * ToDo: re-implement code as appropriate once GHES support is added. * // Make sure SystemConnection Url match Config Url base for OnPremises server * if (!message.Variables.ContainsKey(Constants.Variables.System.ServerType) || * string.Equals(message.Variables[Constants.Variables.System.ServerType]?.Value, "OnPremises", StringComparison.OrdinalIgnoreCase)) * { * try * { * Uri result = null; * Uri configUri = new Uri(_runnerSetting.ServerUrl); * if (Uri.TryCreate(new Uri(configUri.GetComponents(UriComponents.SchemeAndServer, UriFormat.Unescaped)), jobServerUrl.PathAndQuery, out result)) * { * //replace the schema and host portion of messageUri with the host from the * //server URI (which was set at config time) * jobServerUrl = result; * } * } * catch (InvalidOperationException ex) * { * //cannot parse the Uri - not a fatal error * Trace.Error(ex); * } * catch (UriFormatException ex) * { * //cannot parse the Uri - not a fatal error * Trace.Error(ex); * } * } */ await jobServer.ConnectAsync(jobConnection); var timeline = await jobServer.GetTimelineAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, CancellationToken.None); ArgUtil.NotNull(timeline, nameof(timeline)); TimelineRecord jobRecord = timeline.Records.FirstOrDefault(x => x.Id == message.JobId && x.RecordType == "Job"); ArgUtil.NotNull(jobRecord, nameof(jobRecord)); jobRecord.ErrorCount++; jobRecord.Issues.Add(new Issue() { Type = IssueType.Error, Message = errorMessage }); await jobServer.UpdateTimelineRecordsAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, new TimelineRecord[] { jobRecord }, CancellationToken.None); } catch (Exception ex) { Trace.Error("Fail to report unhandled exception from Runner.Worker process"); Trace.Error(ex); } }
public IArtifactDetails GetArtifactDetails(IExecutionContext context, AgentArtifactDefinition agentArtifactDefinition) { ArgUtil.NotNull(context, nameof(context)); ArgUtil.NotNull(agentArtifactDefinition, nameof(agentArtifactDefinition)); Trace.Entering(); ServiceEndpoint vssEndpoint = context.Endpoints.FirstOrDefault(e => string.Equals(e.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); ArgUtil.NotNull(vssEndpoint, nameof(vssEndpoint)); ArgUtil.NotNull(vssEndpoint.Url, nameof(vssEndpoint.Url)); var artifactDetails = JsonConvert.DeserializeObject <Dictionary <string, string> >(agentArtifactDefinition.Details); VssCredentials vssCredentials = VssUtil.GetVssCredential(vssEndpoint); var tfsUrl = context.Variables.Get(WellKnownDistributedTaskVariables.TFCollectionUrl); Guid projectId = context?.Variables.System_TeamProjectId ?? Guid.Empty; if (artifactDetails.ContainsKey("Project")) { Guid.TryParse(artifactDetails["Project"], out projectId); } ArgUtil.NotEmpty(projectId, nameof(projectId)); string relativePath; string accessToken; vssEndpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.AccessToken, out accessToken); if (artifactDetails.TryGetValue("RelativePath", out relativePath)) { var buildArtifactDetails = new BuildArtifactDetails { Credentials = vssCredentials, RelativePath = artifactDetails["RelativePath"], AccessToken = accessToken, Project = projectId.ToString(), TfsUrl = new Uri(tfsUrl) }; if (artifactDetails.ContainsKey("DefinitionName")) { buildArtifactDetails.DefinitionName = artifactDetails["DefinitionName"]; } if (artifactDetails.ContainsKey("DefinitionId")) { buildArtifactDetails.DefintionId = Convert.ToInt32(artifactDetails["DefinitionId"], CultureInfo.InvariantCulture); } return(buildArtifactDetails); } else { throw new InvalidOperationException(StringUtil.Loc("RMArtifactDetailsIncomplete")); } }
public VssConnection InitializeVssConnection() { var headerValues = new List <ProductInfoHeaderValue>(); headerValues.Add(new ProductInfoHeaderValue($"VstsAgentCore-Plugin", Variables.GetValueOrDefault("agent.version")?.Value ?? "Unknown")); headerValues.Add(new ProductInfoHeaderValue($"({RuntimeInformation.OSDescription.Trim()})")); if (VssClientHttpRequestSettings.Default.UserAgent != null && VssClientHttpRequestSettings.Default.UserAgent.Count > 0) { headerValues.AddRange(VssClientHttpRequestSettings.Default.UserAgent); } VssClientHttpRequestSettings.Default.UserAgent = headerValues; if (PlatformUtil.RunningOnLinux || PlatformUtil.RunningOnMacOS) { // The .NET Core 2.1 runtime switched its HTTP default from HTTP 1.1 to HTTP 2. // This causes problems with some versions of the Curl handler. // See GitHub issue https://github.com/dotnet/corefx/issues/32376 VssClientHttpRequestSettings.Default.UseHttp11 = true; } var certSetting = GetCertConfiguration(); if (certSetting != null) { if (!string.IsNullOrEmpty(certSetting.ClientCertificateArchiveFile)) { VssClientHttpRequestSettings.Default.ClientCertificateManager = new AgentClientCertificateManager(certSetting.ClientCertificateArchiveFile, certSetting.ClientCertificatePassword); } if (certSetting.SkipServerCertificateValidation) { VssClientHttpRequestSettings.Default.ServerCertificateValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator; } } var proxySetting = GetProxyConfiguration(); if (proxySetting != null) { if (!string.IsNullOrEmpty(proxySetting.ProxyAddress)) { VssHttpMessageHandler.DefaultWebProxy = new AgentWebProxy(proxySetting.ProxyAddress, proxySetting.ProxyUsername, proxySetting.ProxyPassword, proxySetting.ProxyBypassList); } } ServiceEndpoint systemConnection = this.Endpoints.FirstOrDefault(e => string.Equals(e.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); ArgUtil.NotNull(systemConnection, nameof(systemConnection)); ArgUtil.NotNull(systemConnection.Url, nameof(systemConnection.Url)); VssCredentials credentials = VssUtil.GetVssCredential(systemConnection); ArgUtil.NotNull(credentials, nameof(credentials)); return(VssUtil.CreateConnection(systemConnection.Url, credentials)); }
private VssConnection InitializeVssConnection() { var headerValues = new List <ProductInfoHeaderValue>(); headerValues.Add(new ProductInfoHeaderValue($"VstsAgentCore-Plugin", Variables.GetValueOrDefault("agent.version")?.Value ?? "Unknown")); headerValues.Add(new ProductInfoHeaderValue($"({RuntimeInformation.OSDescription.Trim()})")); if (VssClientHttpRequestSettings.Default.UserAgent != null && VssClientHttpRequestSettings.Default.UserAgent.Count > 0) { headerValues.AddRange(VssClientHttpRequestSettings.Default.UserAgent); } VssClientHttpRequestSettings.Default.UserAgent = headerValues; var certSetting = GetCertConfiguration(); if (certSetting != null) { if (!string.IsNullOrEmpty(certSetting.ClientCertificateArchiveFile)) { VssClientHttpRequestSettings.Default.ClientCertificateManager = new AgentClientCertificateManager(certSetting.ClientCertificateArchiveFile, certSetting.ClientCertificatePassword); } if (certSetting.SkipServerCertificateValidation) { VssClientHttpRequestSettings.Default.ServerCertificateValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator; } } var proxySetting = GetProxyConfiguration(); if (proxySetting != null) { if (!string.IsNullOrEmpty(proxySetting.ProxyAddress)) { VssHttpMessageHandler.DefaultWebProxy = new AgentWebProxy(proxySetting.ProxyAddress, proxySetting.ProxyUsername, proxySetting.ProxyPassword, proxySetting.ProxyBypassList); } } ServiceEndpoint systemConnection = this.Endpoints.FirstOrDefault(e => string.Equals(e.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); ArgUtil.NotNull(systemConnection, nameof(systemConnection)); ArgUtil.NotNull(systemConnection.Url, nameof(systemConnection.Url)); VssCredentials credentials = VssUtil.GetVssCredential(systemConnection); ArgUtil.NotNull(credentials, nameof(credentials)); return(VssUtil.CreateConnection(systemConnection.Url, credentials)); }
public static VssConnection GetVssConnection(IExecutionContext context) { ArgUtil.NotNull(context, nameof(context)); ArgUtil.NotNull(context.Endpoints, nameof(context.Endpoints)); ServiceEndpoint systemConnection = context.Endpoints.FirstOrDefault(e => string.Equals(e.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); ArgUtil.NotNull(systemConnection, nameof(systemConnection)); ArgUtil.NotNull(systemConnection.Url, nameof(systemConnection.Url)); VssCredentials credentials = VssUtil.GetVssCredential(systemConnection); ArgUtil.NotNull(credentials, nameof(credentials)); VssConnection connection = VssUtil.CreateConnection(systemConnection.Url, credentials); return(connection); }
public async Task <TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken) { // Validate parameters. Trace.Entering(); ArgUtil.NotNull(message, nameof(message)); ArgUtil.NotNull(message.Resources, nameof(message.Resources)); ArgUtil.NotNull(message.Variables, nameof(message.Variables)); ArgUtil.NotNull(message.Steps, nameof(message.Steps)); Trace.Info("Job ID {0}", message.JobId); DateTime jobStartTimeUtc = DateTime.UtcNow; ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); // Setup the job server and job server queue. var jobServer = HostContext.GetService <IJobServer>(); VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection); Uri jobServerUrl = systemConnection.Url; Trace.Info($"Creating job server with URL: {jobServerUrl}"); // jobServerQueue is the throttling reporter. _jobServerQueue = HostContext.GetService <IJobServerQueue>(); VssConnection jobConnection = VssUtil.CreateConnection(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(_jobServerQueue) }); await jobServer.ConnectAsync(jobConnection); _jobServerQueue.Start(message); HostContext.WritePerfCounter($"WorkerJobServerQueueStarted_{message.RequestId.ToString()}"); IExecutionContext jobContext = null; CancellationTokenRegistration?runnerShutdownRegistration = null; try { // Create the job execution context. jobContext = HostContext.CreateService <IExecutionContext>(); jobContext.InitializeJob(message, jobRequestCancellationToken); Trace.Info("Starting the job execution context."); jobContext.Start(); jobContext.Debug($"Starting: {message.JobDisplayName}"); runnerShutdownRegistration = HostContext.RunnerShutdownToken.Register(() => { // log an issue, then runner get shutdown by Ctrl-C or Ctrl-Break. // the server will use Ctrl-Break to tells the runner that operating system is shutting down. string errorMessage; switch (HostContext.RunnerShutdownReason) { case ShutdownReason.UserCancelled: errorMessage = "The runner has received a shutdown signal. This can happen when the runner service is stopped, or a manually started runner is canceled."; break; case ShutdownReason.OperatingSystemShutdown: errorMessage = $"Operating system is shutting down for computer '{Environment.MachineName}'"; break; default: throw new ArgumentException(HostContext.RunnerShutdownReason.ToString(), nameof(HostContext.RunnerShutdownReason)); } jobContext.AddIssue(new Issue() { Type = IssueType.Error, Message = errorMessage }); }); // Validate directory permissions. string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work); Trace.Info($"Validating directory permissions for: '{workDirectory}'"); try { Directory.CreateDirectory(workDirectory); IOUtil.ValidateExecutePermission(workDirectory); } catch (Exception ex) { Trace.Error(ex); jobContext.Error(ex); return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed)); } if (jobContext.Global.WriteDebug) { jobContext.SetRunnerContext("debug", "1"); } jobContext.SetRunnerContext("os", VarUtil.OS); string toolsDirectory = HostContext.GetDirectory(WellKnownDirectory.Tools); Directory.CreateDirectory(toolsDirectory); jobContext.SetRunnerContext("tool_cache", toolsDirectory); // Setup TEMP directories _tempDirectoryManager = HostContext.GetService <ITempDirectoryManager>(); _tempDirectoryManager.InitializeTempDirectory(jobContext); // Get the job extension. Trace.Info("Getting job extension."); IJobExtension jobExtension = HostContext.CreateService <IJobExtension>(); List <IStep> jobSteps = null; try { Trace.Info("Initialize job. Getting all job steps."); jobSteps = await jobExtension.InitializeJob(jobContext, message); } catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested) { // set the job to canceled // don't log error issue to job ExecutionContext, since server owns the job level issue Trace.Error($"Job is canceled during initialize."); Trace.Error($"Caught exception: {ex}"); return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled)); } catch (Exception ex) { // set the job to failed. // don't log error issue to job ExecutionContext, since server owns the job level issue Trace.Error($"Job initialize failed."); Trace.Error($"Caught exception from {nameof(jobExtension.InitializeJob)}: {ex}"); return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed)); } // trace out all steps Trace.Info($"Total job steps: {jobSteps.Count}."); Trace.Verbose($"Job steps: '{string.Join(", ", jobSteps.Select(x => x.DisplayName))}'"); HostContext.WritePerfCounter($"WorkerJobInitialized_{message.RequestId.ToString()}"); // Run all job steps Trace.Info("Run all job steps."); var stepsRunner = HostContext.GetService <IStepsRunner>(); try { foreach (var step in jobSteps) { jobContext.JobSteps.Enqueue(step); } await stepsRunner.RunAsync(jobContext); } catch (Exception ex) { // StepRunner should never throw exception out. // End up here mean there is a bug in StepRunner // Log the error and fail the job. Trace.Error($"Caught exception from job steps {nameof(StepsRunner)}: {ex}"); jobContext.Error(ex); return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed)); } finally { Trace.Info("Finalize job."); jobExtension.FinalizeJob(jobContext, message, jobStartTimeUtc); } Trace.Info($"Job result after all job steps finish: {jobContext.Result ?? TaskResult.Succeeded}"); Trace.Info("Completing the job execution context."); return(await CompleteJobAsync(jobServer, jobContext, message)); } finally { if (runnerShutdownRegistration != null) { runnerShutdownRegistration.Value.Dispose(); runnerShutdownRegistration = null; } await ShutdownQueue(throwOnFailure : false); } }
public async Task <TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken) { // Validate parameters. Trace.Entering(); ArgUtil.NotNull(message, nameof(message)); ArgUtil.NotNull(message.Resources, nameof(message.Resources)); ArgUtil.NotNull(message.Variables, nameof(message.Variables)); ArgUtil.NotNull(message.Steps, nameof(message.Steps)); Trace.Info("Job ID {0}", message.JobId); DateTime jobStartTimeUtc = DateTime.UtcNow; // Agent.RunMode RunMode runMode; if (message.Variables.ContainsKey(Constants.Variables.Agent.RunMode) && Enum.TryParse(message.Variables[Constants.Variables.Agent.RunMode].Value, ignoreCase: true, result: out runMode) && runMode == RunMode.Local) { HostContext.RunMode = runMode; } ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); // System.AccessToken if (message.Variables.ContainsKey(Constants.Variables.System.EnableAccessToken) && StringUtil.ConvertToBoolean(message.Variables[Constants.Variables.System.EnableAccessToken].Value)) { message.Variables[Constants.Variables.System.AccessToken] = new VariableValue(systemConnection.Authorization.Parameters["AccessToken"], false); } // back compat TfsServerUrl message.Variables[Constants.Variables.System.TFServerUrl] = systemConnection.Url.AbsoluteUri; // Make sure SystemConnection Url and Endpoint Url match Config Url base for OnPremises server // System.ServerType will always be there after M133 if (!message.Variables.ContainsKey(Constants.Variables.System.ServerType) || string.Equals(message.Variables[Constants.Variables.System.ServerType]?.Value, "OnPremises", StringComparison.OrdinalIgnoreCase)) { ReplaceConfigUriBaseInJobRequestMessage(message); } // Setup the job server and job server queue. var jobServer = HostContext.GetService <IJobServer>(); VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection); Uri jobServerUrl = systemConnection.Url; Trace.Info($"Creating job server with URL: {jobServerUrl}"); // jobServerQueue is the throttling reporter. _jobServerQueue = HostContext.GetService <IJobServerQueue>(); VssConnection jobConnection = VssUtil.CreateConnection(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(_jobServerQueue) }); await jobServer.ConnectAsync(jobConnection); _jobServerQueue.Start(message); HostContext.WritePerfCounter($"WorkerJobServerQueueStarted_{message.RequestId.ToString()}"); IExecutionContext jobContext = null; CancellationTokenRegistration?agentShutdownRegistration = null; try { // Create the job execution context. jobContext = HostContext.CreateService <IExecutionContext>(); jobContext.InitializeJob(message, jobRequestCancellationToken); Trace.Info("Starting the job execution context."); jobContext.Start(); jobContext.Section(StringUtil.Loc("StepStarting", message.JobDisplayName)); agentShutdownRegistration = HostContext.AgentShutdownToken.Register(() => { // log an issue, then agent get shutdown by Ctrl-C or Ctrl-Break. // the server will use Ctrl-Break to tells the agent that operating system is shutting down. string errorMessage; switch (HostContext.AgentShutdownReason) { case ShutdownReason.UserCancelled: errorMessage = StringUtil.Loc("UserShutdownAgent"); break; case ShutdownReason.OperatingSystemShutdown: errorMessage = StringUtil.Loc("OperatingSystemShutdown", Environment.MachineName); break; default: throw new ArgumentException(HostContext.AgentShutdownReason.ToString(), nameof(HostContext.AgentShutdownReason)); } jobContext.AddIssue(new Issue() { Type = IssueType.Error, Message = errorMessage }); }); // Validate directory permissions. string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work); Trace.Info($"Validating directory permissions for: '{workDirectory}'"); try { Directory.CreateDirectory(workDirectory); IOUtil.ValidateExecutePermission(workDirectory); } catch (Exception ex) { Trace.Error(ex); jobContext.Error(ex); return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed)); } // Set agent variables. AgentSettings settings = HostContext.GetService <IConfigurationStore>().GetSettings(); jobContext.Variables.Set(Constants.Variables.Agent.Id, settings.AgentId.ToString(CultureInfo.InvariantCulture)); jobContext.SetVariable(Constants.Variables.Agent.HomeDirectory, HostContext.GetDirectory(WellKnownDirectory.Root), isFilePath: true); jobContext.Variables.Set(Constants.Variables.Agent.JobName, message.JobDisplayName); jobContext.Variables.Set(Constants.Variables.Agent.MachineName, Environment.MachineName); jobContext.Variables.Set(Constants.Variables.Agent.Name, settings.AgentName); jobContext.Variables.Set(Constants.Variables.Agent.OS, VarUtil.OS); jobContext.Variables.Set(Constants.Variables.Agent.OSArchitecture, VarUtil.OSArchitecture); jobContext.SetVariable(Constants.Variables.Agent.RootDirectory, HostContext.GetDirectory(WellKnownDirectory.Work), isFilePath: true); #if OS_WINDOWS jobContext.SetVariable(Constants.Variables.Agent.ServerOMDirectory, HostContext.GetDirectory(WellKnownDirectory.ServerOM), isFilePath: true); #else jobContext.Variables.Set(Constants.Variables.Agent.AcceptTeeEula, settings.AcceptTeeEula.ToString()); #endif jobContext.SetVariable(Constants.Variables.Agent.WorkFolder, HostContext.GetDirectory(WellKnownDirectory.Work), isFilePath: true); jobContext.SetVariable(Constants.Variables.System.WorkFolder, HostContext.GetDirectory(WellKnownDirectory.Work), isFilePath: true); string toolsDirectory = HostContext.GetDirectory(WellKnownDirectory.Tools); Directory.CreateDirectory(toolsDirectory); jobContext.SetVariable(Constants.Variables.Agent.ToolsDirectory, toolsDirectory, isFilePath: true); // Setup TEMP directories _tempDirectoryManager = HostContext.GetService <ITempDirectoryManager>(); _tempDirectoryManager.InitializeTempDirectory(jobContext); // todo: task server can throw. try/catch and fail job gracefully. // prefer task definitions url, then TFS collection url, then TFS account url var taskServer = HostContext.GetService <ITaskServer>(); Uri taskServerUri = null; if (!string.IsNullOrEmpty(jobContext.Variables.System_TaskDefinitionsUri)) { taskServerUri = new Uri(jobContext.Variables.System_TaskDefinitionsUri); } else if (!string.IsNullOrEmpty(jobContext.Variables.System_TFCollectionUrl)) { taskServerUri = new Uri(jobContext.Variables.System_TFCollectionUrl); } var taskServerCredential = VssUtil.GetVssCredential(systemConnection); if (taskServerUri != null) { Trace.Info($"Creating task server with {taskServerUri}"); await taskServer.ConnectAsync(VssUtil.CreateConnection(taskServerUri, taskServerCredential)); } // for back compat TFS 2015 RTM/QU1, we may need to switch the task server url to agent config url if (!string.Equals(message.Variables.GetValueOrDefault(Constants.Variables.System.ServerType)?.Value, "Hosted", StringComparison.OrdinalIgnoreCase)) { if (taskServerUri == null || !await taskServer.TaskDefinitionEndpointExist()) { Trace.Info($"Can't determine task download url from JobMessage or the endpoint doesn't exist."); var configStore = HostContext.GetService <IConfigurationStore>(); taskServerUri = new Uri(configStore.GetSettings().ServerUrl); Trace.Info($"Recreate task server with configuration server url: {taskServerUri}"); await taskServer.ConnectAsync(VssUtil.CreateConnection(taskServerUri, taskServerCredential)); } } // Expand the endpoint data values. foreach (ServiceEndpoint endpoint in jobContext.Endpoints) { jobContext.Variables.ExpandValues(target: endpoint.Data); VarUtil.ExpandEnvironmentVariables(HostContext, target: endpoint.Data); } // Expand the repository property values. foreach (var repository in jobContext.Repositories) { // expand checkout option var checkoutOptions = repository.Properties.Get <JToken>(Pipelines.RepositoryPropertyNames.CheckoutOptions); if (checkoutOptions != null) { checkoutOptions = jobContext.Variables.ExpandValues(target: checkoutOptions); checkoutOptions = VarUtil.ExpandEnvironmentVariables(HostContext, target: checkoutOptions); repository.Properties.Set <JToken>(Pipelines.RepositoryPropertyNames.CheckoutOptions, checkoutOptions);; } // expand workspace mapping var mappings = repository.Properties.Get <JToken>(Pipelines.RepositoryPropertyNames.Mappings); if (mappings != null) { mappings = jobContext.Variables.ExpandValues(target: mappings); mappings = VarUtil.ExpandEnvironmentVariables(HostContext, target: mappings); repository.Properties.Set <JToken>(Pipelines.RepositoryPropertyNames.Mappings, mappings); } } // Expand container properties jobContext.Container?.ExpandProperties(jobContext.Variables); foreach (var sidecar in jobContext.SidecarContainers) { sidecar.ExpandProperties(jobContext.Variables); } // Get the job extension. Trace.Info("Getting job extension."); var hostType = jobContext.Variables.System_HostType; var extensionManager = HostContext.GetService <IExtensionManager>(); // We should always have one job extension IJobExtension jobExtension = (extensionManager.GetExtensions <IJobExtension>() ?? new List <IJobExtension>()) .Where(x => x.HostType.HasFlag(hostType)) .FirstOrDefault(); ArgUtil.NotNull(jobExtension, nameof(jobExtension)); List <IStep> jobSteps = new List <IStep>(); try { Trace.Info("Initialize job. Getting all job steps."); var initializeResult = await jobExtension.InitializeJob(jobContext, message); jobSteps.AddRange(initializeResult.PreJobSteps); jobSteps.AddRange(initializeResult.JobSteps); jobSteps.AddRange(initializeResult.PostJobStep); } catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested) { // set the job to canceled // don't log error issue to job ExecutionContext, since server owns the job level issue Trace.Error($"Job is canceled during initialize."); Trace.Error($"Caught exception: {ex}"); return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled)); } catch (Exception ex) { // set the job to failed. // don't log error issue to job ExecutionContext, since server owns the job level issue Trace.Error($"Job initialize failed."); Trace.Error($"Caught exception from {nameof(jobExtension.InitializeJob)}: {ex}"); return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed)); } // trace out all steps Trace.Info($"Total job steps: {jobSteps.Count}."); Trace.Verbose($"Job steps: '{string.Join(", ", jobSteps.Select(x => x.DisplayName))}'"); HostContext.WritePerfCounter($"WorkerJobInitialized_{message.RequestId.ToString()}"); bool processCleanup = jobContext.Variables.GetBoolean("process.clean") ?? true; HashSet <string> existingProcesses = new HashSet <string>(StringComparer.OrdinalIgnoreCase); string processLookupId = null; if (processCleanup) { processLookupId = $"vsts_{Guid.NewGuid()}"; // Set the VSTS_PROCESS_LOOKUP_ID env variable. jobContext.SetVariable(Constants.ProcessLookupId, processLookupId, false, false); // Take a snapshot of current running processes Dictionary <int, Process> processes = SnapshotProcesses(); foreach (var proc in processes) { // Pid_ProcessName existingProcesses.Add($"{proc.Key}_{proc.Value.ProcessName}"); } } // Run all job steps Trace.Info("Run all job steps."); var stepsRunner = HostContext.GetService <IStepsRunner>(); try { await stepsRunner.RunAsync(jobContext, jobSteps); } catch (Exception ex) { // StepRunner should never throw exception out. // End up here mean there is a bug in StepRunner // Log the error and fail the job. Trace.Error($"Caught exception from job steps {nameof(StepsRunner)}: {ex}"); jobContext.Error(ex); return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed)); } finally { if (processCleanup) { // Only check environment variable for any process that doesn't run before we invoke our process. Dictionary <int, Process> currentProcesses = SnapshotProcesses(); foreach (var proc in currentProcesses) { if (existingProcesses.Contains($"{proc.Key}_{proc.Value.ProcessName}")) { Trace.Verbose($"Skip existing process. PID: {proc.Key} ({proc.Value.ProcessName})"); } else { Trace.Info($"Inspecting process environment variables. PID: {proc.Key} ({proc.Value.ProcessName})"); string lookupId = null; try { lookupId = proc.Value.GetEnvironmentVariable(HostContext, Constants.ProcessLookupId); } catch (Exception ex) { Trace.Warning($"Ignore exception during read process environment variables: {ex.Message}"); Trace.Verbose(ex.ToString()); } if (string.Equals(lookupId, processLookupId, StringComparison.OrdinalIgnoreCase)) { Trace.Info($"Terminate orphan process: pid ({proc.Key}) ({proc.Value.ProcessName})"); try { proc.Value.Kill(); } catch (Exception ex) { Trace.Error("Catch exception during orphan process cleanup."); Trace.Error(ex); } } } } } } Trace.Info($"Job result after all job steps finish: {jobContext.Result ?? TaskResult.Succeeded}"); if (jobContext.Variables.GetBoolean(Constants.Variables.Agent.Diagnostic) ?? false) { Trace.Info("Support log upload starting."); IDiagnosticLogManager diagnosticLogManager = HostContext.GetService <IDiagnosticLogManager>(); try { await diagnosticLogManager.UploadDiagnosticLogsAsync(executionContext : jobContext, message : message, jobStartTimeUtc : jobStartTimeUtc); Trace.Info("Support log upload complete."); } catch (Exception ex) { // Log the error but make sure we continue gracefully. Trace.Info("Error uploading support logs."); Trace.Error(ex); } } Trace.Info("Completing the job execution context."); return(await CompleteJobAsync(jobServer, jobContext, message)); } finally { if (agentShutdownRegistration != null) { agentShutdownRegistration.Value.Dispose(); agentShutdownRegistration = null; } await ShutdownQueue(throwOnFailure : false); } }
public void Execute(IExecutionContext context, Command command) { ArgUtil.NotNull(context, nameof(context)); ArgUtil.NotNull(context.Endpoints, nameof(context.Endpoints)); var eventProperties = command.Properties; var data = command.Data; ServiceEndpoint systemConnection = context.Endpoints.FirstOrDefault(e => string.Equals(e.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); ArgUtil.NotNull(systemConnection, nameof(systemConnection)); ArgUtil.NotNull(systemConnection.Url, nameof(systemConnection.Url)); Uri projectUrl = systemConnection.Url; VssCredentials projectCredential = VssUtil.GetVssCredential(systemConnection); Guid projectId = context.Variables.System_TeamProjectId ?? Guid.Empty; ArgUtil.NotEmpty(projectId, nameof(projectId)); int?buildId = context.Variables.Build_BuildId; ArgUtil.NotNull(buildId, nameof(buildId)); string artifactName; if (!eventProperties.TryGetValue(ArtifactAssociateEventProperties.ArtifactName, out artifactName) || string.IsNullOrEmpty(artifactName)) { throw new Exception(StringUtil.Loc("ArtifactNameRequired")); } string artifactType; if (!eventProperties.TryGetValue(ArtifactAssociateEventProperties.ArtifactType, out artifactType)) { artifactType = ArtifactCommandExtensionUtil.InferArtifactResourceType(context, data); } if (string.IsNullOrEmpty(artifactType)) { throw new Exception(StringUtil.Loc("ArtifactTypeRequired")); } else if ((artifactType.Equals(ArtifactResourceTypes.Container, StringComparison.OrdinalIgnoreCase) || artifactType.Equals(ArtifactResourceTypes.FilePath, StringComparison.OrdinalIgnoreCase) || artifactType.Equals(ArtifactResourceTypes.VersionControl, StringComparison.OrdinalIgnoreCase)) && string.IsNullOrEmpty(data)) { throw new Exception(StringUtil.Loc("ArtifactLocationRequired")); } if (!artifactType.Equals(ArtifactResourceTypes.FilePath, StringComparison.OrdinalIgnoreCase) && context.Variables.System_HostType != HostTypes.Build) { throw new Exception(StringUtil.Loc("AssociateArtifactCommandNotSupported", context.Variables.System_HostType)); } var propertyDictionary = ArtifactCommandExtensionUtil.ExtractArtifactProperties(eventProperties); string artifactData = ""; if (ArtifactCommandExtensionUtil.IsContainerPath(data) || ArtifactCommandExtensionUtil.IsValidServerPath(data)) { //if data is a file container path or a tfvc server path artifactData = data; } else if (ArtifactCommandExtensionUtil.IsUncSharePath(context, data)) { //if data is a UNC share path artifactData = new Uri(data).LocalPath; } else { artifactData = data ?? string.Empty; } // queue async command task to associate artifact. context.Debug($"Associate artifact: {artifactName} with build: {buildId.Value} at backend."); var commandContext = context.GetHostContext().CreateService <IAsyncCommandContext>(); commandContext.InitializeCommandContext(context, StringUtil.Loc("AssociateArtifact")); commandContext.Task = ArtifactCommandExtensionUtil.AssociateArtifactAsync(commandContext, WorkerUtilities.GetVssConnection(context), projectId, buildId.Value, artifactName, context.Variables.System_JobId, artifactType, artifactData, propertyDictionary, context.CancellationToken); context.AsyncCommands.Add(commandContext); }
public async Task <TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken) { // Validate parameters. Trace.Entering(); ArgUtil.NotNull(message, nameof(message)); ArgUtil.NotNull(message.Resources, nameof(message.Resources)); ArgUtil.NotNull(message.Variables, nameof(message.Variables)); ArgUtil.NotNull(message.Steps, nameof(message.Steps)); Trace.Info("Job ID {0}", message.JobId); DateTime jobStartTimeUtc = DateTime.UtcNow; ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); // Setup the job server and job server queue. var jobServer = HostContext.GetService <IJobServer>(); VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection); Uri jobServerUrl = systemConnection.Url; Trace.Info($"Creating job server with URL: {jobServerUrl}"); // jobServerQueue is the throttling reporter. _jobServerQueue = HostContext.GetService <IJobServerQueue>(); VssConnection jobConnection = VssUtil.CreateConnection(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(_jobServerQueue) }); await jobServer.ConnectAsync(jobConnection); _jobServerQueue.Start(message); HostContext.WritePerfCounter($"WorkerJobServerQueueStarted_{message.RequestId.ToString()}"); IExecutionContext jobContext = null; CancellationTokenRegistration?runnerShutdownRegistration = null; try { // Create the job execution context. jobContext = HostContext.CreateService <IExecutionContext>(); jobContext.InitializeJob(message, jobRequestCancellationToken); Trace.Info("Starting the job execution context."); jobContext.Start(); jobContext.Debug($"Starting: {message.JobDisplayName}"); // RUST: If the event type is not allowed exit the job before anything is run. var rustExpectedEvent = System.Environment.GetEnvironmentVariable("RUST_WHITELISTED_EVENT_NAME"); if (rustExpectedEvent != null) { var rustGitHubContext = (Pipelines.ContextData.DictionaryContextData)message.ContextData["github"]; var rustEventName = rustGitHubContext["event_name"].ToString(); if (rustEventName != rustExpectedEvent) { return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled)); } } runnerShutdownRegistration = HostContext.RunnerShutdownToken.Register(() => { // log an issue, then runner get shutdown by Ctrl-C or Ctrl-Break. // the server will use Ctrl-Break to tells the runner that operating system is shutting down. string errorMessage; switch (HostContext.RunnerShutdownReason) { case ShutdownReason.UserCancelled: errorMessage = "The runner has received a shutdown signal. This can happen when the runner service is stopped, or a manually started runner is canceled."; break; case ShutdownReason.OperatingSystemShutdown: errorMessage = $"Operating system is shutting down for computer '{Environment.MachineName}'"; break; default: throw new ArgumentException(HostContext.RunnerShutdownReason.ToString(), nameof(HostContext.RunnerShutdownReason)); } jobContext.AddIssue(new Issue() { Type = IssueType.Error, Message = errorMessage }); }); // Validate directory permissions. string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work); Trace.Info($"Validating directory permissions for: '{workDirectory}'"); try { Directory.CreateDirectory(workDirectory); IOUtil.ValidateExecutePermission(workDirectory); } catch (Exception ex) { Trace.Error(ex); jobContext.Error(ex); return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed)); } if (jobContext.Global.WriteDebug) { jobContext.SetRunnerContext("debug", "1"); } jobContext.SetRunnerContext("os", VarUtil.OS); jobContext.SetRunnerContext("arch", VarUtil.OSArchitecture); var runnerSettings = HostContext.GetService <IConfigurationStore>().GetSettings(); jobContext.SetRunnerContext("name", runnerSettings.AgentName); string toolsDirectory = HostContext.GetDirectory(WellKnownDirectory.Tools); Directory.CreateDirectory(toolsDirectory); jobContext.SetRunnerContext("tool_cache", toolsDirectory); // Setup TEMP directories _tempDirectoryManager = HostContext.GetService <ITempDirectoryManager>(); _tempDirectoryManager.InitializeTempDirectory(jobContext); // Get the job extension. Trace.Info("Getting job extension."); IJobExtension jobExtension = HostContext.CreateService <IJobExtension>(); List <IStep> jobSteps = null; try { Trace.Info("Initialize job. Getting all job steps."); jobSteps = await jobExtension.InitializeJob(jobContext, message); } catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested) { // set the job to canceled // don't log error issue to job ExecutionContext, since server owns the job level issue Trace.Error($"Job is canceled during initialize."); Trace.Error($"Caught exception: {ex}"); return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled)); } catch (Exception ex) { // set the job to failed. // don't log error issue to job ExecutionContext, since server owns the job level issue Trace.Error($"Job initialize failed."); Trace.Error($"Caught exception from {nameof(jobExtension.InitializeJob)}: {ex}"); return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed)); } // trace out all steps Trace.Info($"Total job steps: {jobSteps.Count}."); Trace.Verbose($"Job steps: '{string.Join(", ", jobSteps.Select(x => x.DisplayName))}'"); HostContext.WritePerfCounter($"WorkerJobInitialized_{message.RequestId.ToString()}"); if (systemConnection.Data.TryGetValue("GenerateIdTokenUrl", out var generateIdTokenUrl) && !string.IsNullOrEmpty(generateIdTokenUrl)) { // Server won't issue ID_TOKEN for non-inprogress job. // If the job is trying to use OIDC feature, we want the job to be marked as in-progress before running any customer's steps as much as we can. // Timeline record update background process runs every 500ms, so delay 1000ms is enough for most of the cases Trace.Info($"Waiting for job to be marked as started."); await Task.WhenAny(_jobServerQueue.JobRecordUpdated.Task, Task.Delay(1000)); } // Run all job steps Trace.Info("Run all job steps."); var stepsRunner = HostContext.GetService <IStepsRunner>(); try { foreach (var step in jobSteps) { jobContext.JobSteps.Enqueue(step); } await stepsRunner.RunAsync(jobContext); } catch (Exception ex) { // StepRunner should never throw exception out. // End up here mean there is a bug in StepRunner // Log the error and fail the job. Trace.Error($"Caught exception from job steps {nameof(StepsRunner)}: {ex}"); jobContext.Error(ex); return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed)); } finally { Trace.Info("Finalize job."); jobExtension.FinalizeJob(jobContext, message, jobStartTimeUtc); } Trace.Info($"Job result after all job steps finish: {jobContext.Result ?? TaskResult.Succeeded}"); Trace.Info("Completing the job execution context."); return(await CompleteJobAsync(jobServer, jobContext, message)); } finally { if (runnerShutdownRegistration != null) { runnerShutdownRegistration.Value.Dispose(); runnerShutdownRegistration = null; } await ShutdownQueue(throwOnFailure : false); } }