예제 #1
0
        public void AddIssue_CountWarningsErrors()
        {
            using (TestHostContext hc = CreateTestContext())
            {
                // Arrange: Create a job request message.
                TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
                TimelineReference timeline          = new TimelineReference();
                Guid   jobId      = Guid.NewGuid();
                string jobName    = "some job name";
                var    jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary <string, VariableValue>(), new List <MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List <Pipelines.ActionStep>(), null, null, null, null);
                jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                {
                    Alias   = Pipelines.PipelineConstants.SelfAlias,
                    Id      = "github",
                    Version = "sha1"
                });
                jobRequest.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData();

                // Arrange: Setup the paging logger.
                var pagingLogger   = new Mock <IPagingLogger>();
                var jobServerQueue = new Mock <IJobServerQueue>();
                jobServerQueue.Setup(x => x.QueueTimelineRecordUpdate(It.IsAny <Guid>(), It.IsAny <TimelineRecord>()));

                hc.EnqueueInstance(pagingLogger.Object);
                hc.SetSingleton(jobServerQueue.Object);

                var ec = new Runner.Worker.ExecutionContext();
                ec.Initialize(hc);

                // Act.
                ec.InitializeJob(jobRequest, CancellationToken.None);

                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Error, Message = "error"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Error, Message = "error"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Error, Message = "error"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Error, Message = "error"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Error, Message = "error"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Error, Message = "error"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Error, Message = "error"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Error, Message = "error"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Error, Message = "error"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Error, Message = "error"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Error, Message = "error"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Error, Message = "error"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Error, Message = "error"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Error, Message = "error"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Error, Message = "error"
                });

                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Warning, Message = "warning"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Warning, Message = "warning"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Warning, Message = "warning"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Warning, Message = "warning"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Warning, Message = "warning"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Warning, Message = "warning"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Warning, Message = "warning"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Warning, Message = "warning"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Warning, Message = "warning"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Warning, Message = "warning"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Warning, Message = "warning"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Warning, Message = "warning"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Warning, Message = "warning"
                });
                ec.AddIssue(new Issue()
                {
                    Type = IssueType.Warning, Message = "warning"
                });

                ec.Complete();

                // Assert.
                jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny <Guid>(), It.Is <TimelineRecord>(t => t.ErrorCount == 15)), Times.AtLeastOnce);
                jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny <Guid>(), It.Is <TimelineRecord>(t => t.WarningCount == 14)), Times.AtLeastOnce);
                jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny <Guid>(), It.Is <TimelineRecord>(t => t.Issues.Where(i => i.Type == IssueType.Error).Count() == 10)), Times.AtLeastOnce);
                jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny <Guid>(), It.Is <TimelineRecord>(t => t.Issues.Where(i => i.Type == IssueType.Warning).Count() == 10)), Times.AtLeastOnce);
            }
        }
예제 #2
0
        public void FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, DateTime jobStartTimeUtc)
        {
            Trace.Entering();
            ArgUtil.NotNull(jobContext, nameof(jobContext));

            // create a new timeline record node for 'Finalize job'
            IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), "Complete job", $"{nameof(JobExtension)}_Final", null, null);

            using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
            {
                try
                {
                    context.Start();
                    context.Debug("Starting: Complete job");

                    // Evaluate job outputs
                    if (message.JobOutputs != null && message.JobOutputs.Type != TokenType.Null)
                    {
                        try
                        {
                            context.Output($"Evaluate and set job outputs");

                            // Populate env context for each step
                            Trace.Info("Initialize Env context for evaluating job outputs");
#if OS_WINDOWS
                            var envContext = new DictionaryContextData();
#else
                            var envContext = new CaseSensitiveDictionaryContextData();
#endif
                            context.ExpressionValues["env"] = envContext;
                            foreach (var pair in context.EnvironmentVariables)
                            {
                                envContext[pair.Key] = new StringContextData(pair.Value ?? string.Empty);
                            }

                            Trace.Info("Initialize steps context for evaluating job outputs");
                            context.ExpressionValues["steps"] = context.StepsContext.GetScope(context.ScopeName);

                            var templateEvaluator = context.ToPipelineTemplateEvaluator();
                            var outputs           = templateEvaluator.EvaluateJobOutput(message.JobOutputs, context.ExpressionValues, context.ExpressionFunctions);
                            foreach (var output in outputs)
                            {
                                if (string.IsNullOrEmpty(output.Value))
                                {
                                    context.Debug($"Skip output '{output.Key}' since it's empty");
                                    continue;
                                }

                                if (!string.Equals(output.Value, HostContext.SecretMasker.MaskSecrets(output.Value)))
                                {
                                    context.Warning($"Skip output '{output.Key}' since it may contain secret.");
                                    continue;
                                }

                                context.Output($"Set output '{output.Key}'");
                                jobContext.JobOutputs[output.Key] = output.Value;
                            }
                        }
                        catch (Exception ex)
                        {
                            context.Result = TaskResult.Failed;
                            context.Error($"Fail to evaluate job outputs");
                            context.Error(ex);
                            jobContext.Result = TaskResultUtil.MergeTaskResults(jobContext.Result, TaskResult.Failed);
                        }
                    }

                    if (context.Variables.GetBoolean(Constants.Variables.Actions.RunnerDebug) ?? false)
                    {
                        Trace.Info("Support log upload starting.");
                        context.Output("Uploading runner diagnostic logs");

                        IDiagnosticLogManager diagnosticLogManager = HostContext.GetService <IDiagnosticLogManager>();

                        try
                        {
                            diagnosticLogManager.UploadDiagnosticLogs(executionContext: context, parentContext: jobContext, message: message, jobStartTimeUtc: jobStartTimeUtc);

                            Trace.Info("Support log upload complete.");
                            context.Output("Completed runner diagnostic log upload");
                        }
                        catch (Exception ex)
                        {
                            // Log the error but make sure we continue gracefully.
                            Trace.Info("Error uploading support logs.");
                            context.Output("Error uploading runner diagnostic logs");
                            Trace.Error(ex);
                        }
                    }

                    if (_processCleanup)
                    {
                        context.Output("Cleaning up orphan processes");

                        // Only check environment variable for any process that doesn't run before we invoke our process.
                        Dictionary <int, Process> currentProcesses = SnapshotProcesses();
                        foreach (var proc in currentProcesses)
                        {
                            if (proc.Key == Process.GetCurrentProcess().Id)
                            {
                                // skip for current process.
                                continue;
                            }

                            if (_existingProcesses.Contains($"{proc.Key}_{proc.Value.ProcessName}"))
                            {
                                Trace.Verbose($"Skip existing process. PID: {proc.Key} ({proc.Value.ProcessName})");
                            }
                            else
                            {
                                Trace.Info($"Inspecting process environment variables. PID: {proc.Key} ({proc.Value.ProcessName})");

                                string lookupId = null;
                                try
                                {
                                    lookupId = proc.Value.GetEnvironmentVariable(HostContext, Constants.ProcessTrackingId);
                                }
                                catch (Exception ex)
                                {
                                    Trace.Warning($"Ignore exception during read process environment variables: {ex.Message}");
                                    Trace.Verbose(ex.ToString());
                                }

                                if (string.Equals(lookupId, _processLookupId, StringComparison.OrdinalIgnoreCase))
                                {
                                    context.Output($"Terminate orphan process: pid ({proc.Key}) ({proc.Value.ProcessName})");
                                    try
                                    {
                                        proc.Value.Kill();
                                    }
                                    catch (Exception ex)
                                    {
                                        Trace.Error("Catch exception during orphan process cleanup.");
                                        Trace.Error(ex);
                                    }
                                }
                            }
                        }
                    }
                }
                catch (Exception ex)
                {
                    // Log and ignore the error from JobExtension finalization.
                    Trace.Error($"Caught exception from JobExtension finalization: {ex}");
                    context.Output(ex.Message);
                }
                finally
                {
                    context.Debug("Finishing: Complete job");
                    context.Complete();
                }
            }
        }
예제 #3
0
        // Download all required actions.
        // Make sure all condition inputs are valid.
        // Build up three list of steps for jobrunner (pre-job, job, post-job).
        public async Task <List <IStep> > InitializeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message)
        {
            Trace.Entering();
            ArgUtil.NotNull(jobContext, nameof(jobContext));
            ArgUtil.NotNull(message, nameof(message));

            // Create a new timeline record for 'Set up job'
            IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), "Set up job", $"{nameof(JobExtension)}_Init", null, null);

            List <IStep> preJobSteps = new List <IStep>();
            List <IStep> jobSteps    = new List <IStep>();

            using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
            {
                try
                {
                    context.Start();
                    context.Debug($"Starting: Set up job");
                    context.Output($"Current runner version: '{BuildConstants.RunnerPackage.Version}'");

                    var setupInfoFile = HostContext.GetConfigFile(WellKnownConfigFile.SetupInfo);
                    if (File.Exists(setupInfoFile))
                    {
                        Trace.Info($"Load machine setup info from {setupInfoFile}");
                        try
                        {
                            var setupInfo = IOUtil.LoadObject <List <SetupInfo> >(setupInfoFile);
                            if (setupInfo?.Count > 0)
                            {
                                foreach (var info in setupInfo)
                                {
                                    if (!string.IsNullOrEmpty(info?.Detail))
                                    {
                                        var groupName = info.Group;
                                        if (string.IsNullOrEmpty(groupName))
                                        {
                                            groupName = "Machine Setup Info";
                                        }

                                        context.Output($"##[group]{groupName}");
                                        var multiLines = info.Detail.Replace("\r\n", "\n").TrimEnd('\n').Split('\n');
                                        foreach (var line in multiLines)
                                        {
                                            context.Output(line);
                                        }
                                        context.Output("##[endgroup]");
                                    }
                                }
                            }
                        }
                        catch (Exception ex)
                        {
                            context.Output($"Fail to load and print machine setup info: {ex.Message}");
                            Trace.Error(ex);
                        }
                    }

                    var repoFullName = context.GetGitHubContext("repository");
                    ArgUtil.NotNull(repoFullName, nameof(repoFullName));
                    context.Debug($"Primary repository: {repoFullName}");

                    // Print proxy setting information for better diagnostic experience
                    if (!string.IsNullOrEmpty(HostContext.WebProxy.HttpProxyAddress))
                    {
                        context.Output($"Runner is running behind proxy server '{HostContext.WebProxy.HttpProxyAddress}' for all HTTP requests.");
                    }
                    if (!string.IsNullOrEmpty(HostContext.WebProxy.HttpsProxyAddress))
                    {
                        context.Output($"Runner is running behind proxy server '{HostContext.WebProxy.HttpsProxyAddress}' for all HTTPS requests.");
                    }

                    // Prepare the workflow directory
                    context.Output("Prepare workflow directory");
                    var            directoryManager = HostContext.GetService <IPipelineDirectoryManager>();
                    TrackingConfig trackingConfig   = directoryManager.PrepareDirectory(
                        context,
                        message.Workspace);

                    // Set the directory variables
                    context.Debug("Update context data");
                    string _workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
                    context.SetRunnerContext("workspace", Path.Combine(_workDirectory, trackingConfig.PipelineDirectory));
                    context.SetGitHubContext("workspace", Path.Combine(_workDirectory, trackingConfig.WorkspaceDirectory));

                    // Temporary hack for GHES alpha
                    var configurationStore = HostContext.GetService <IConfigurationStore>();
                    var runnerSettings     = configurationStore.GetSettings();
                    if (string.IsNullOrEmpty(context.GetGitHubContext("server_url")) && !runnerSettings.IsHostedServer && !string.IsNullOrEmpty(runnerSettings.GitHubUrl))
                    {
                        var url      = new Uri(runnerSettings.GitHubUrl);
                        var portInfo = url.IsDefaultPort ? string.Empty : $":{url.Port.ToString(CultureInfo.InvariantCulture)}";
                        context.SetGitHubContext("server_url", $"{url.Scheme}://{url.Host}{portInfo}");
                        context.SetGitHubContext("api_url", $"{url.Scheme}://{url.Host}{portInfo}/api/v3");
                        context.SetGitHubContext("graphql_url", $"{url.Scheme}://{url.Host}{portInfo}/api/graphql");
                    }

                    // Evaluate the job-level environment variables
                    context.Debug("Evaluating job-level environment variables");
                    var templateEvaluator = context.ToPipelineTemplateEvaluator();
                    foreach (var token in message.EnvironmentVariables)
                    {
                        var environmentVariables = templateEvaluator.EvaluateStepEnvironment(token, jobContext.ExpressionValues, jobContext.ExpressionFunctions, VarUtil.EnvironmentVariableKeyComparer);
                        foreach (var pair in environmentVariables)
                        {
                            context.EnvironmentVariables[pair.Key] = pair.Value ?? string.Empty;
                            context.SetEnvContext(pair.Key, pair.Value ?? string.Empty);
                        }
                    }

                    // Evaluate the job container
                    context.Debug("Evaluating job container");
                    var container = templateEvaluator.EvaluateJobContainer(message.JobContainer, jobContext.ExpressionValues, jobContext.ExpressionFunctions);
                    if (container != null)
                    {
                        jobContext.Container = new Container.ContainerInfo(HostContext, container);
                    }

                    // Evaluate the job service containers
                    context.Debug("Evaluating job service containers");
                    var serviceContainers = templateEvaluator.EvaluateJobServiceContainers(message.JobServiceContainers, jobContext.ExpressionValues, jobContext.ExpressionFunctions);
                    if (serviceContainers?.Count > 0)
                    {
                        foreach (var pair in serviceContainers)
                        {
                            var networkAlias     = pair.Key;
                            var serviceContainer = pair.Value;
                            jobContext.ServiceContainers.Add(new Container.ContainerInfo(HostContext, serviceContainer, false, networkAlias));
                        }
                    }

                    // Evaluate the job defaults
                    context.Debug("Evaluating job defaults");
                    foreach (var token in message.Defaults)
                    {
                        var defaults = token.AssertMapping("defaults");
                        if (defaults.Any(x => string.Equals(x.Key.AssertString("defaults key").Value, "run", StringComparison.OrdinalIgnoreCase)))
                        {
                            context.JobDefaults["run"] = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
                            var defaultsRun = defaults.First(x => string.Equals(x.Key.AssertString("defaults key").Value, "run", StringComparison.OrdinalIgnoreCase));
                            var jobDefaults = templateEvaluator.EvaluateJobDefaultsRun(defaultsRun.Value, jobContext.ExpressionValues, jobContext.ExpressionFunctions);
                            foreach (var pair in jobDefaults)
                            {
                                if (!string.IsNullOrEmpty(pair.Value))
                                {
                                    context.JobDefaults["run"][pair.Key] = pair.Value;
                                }
                            }
                        }
                    }

                    // Build up 2 lists of steps, pre-job, job
                    // Download actions not already in the cache
                    Trace.Info("Downloading actions");
                    var actionManager = HostContext.GetService <IActionManager>();
                    var prepareResult = await actionManager.PrepareActionsAsync(context, message.Steps);

                    preJobSteps.AddRange(prepareResult.ContainerSetupSteps);

                    // Add start-container steps, record and stop-container steps
                    if (jobContext.Container != null || jobContext.ServiceContainers.Count > 0)
                    {
                        var containerProvider = HostContext.GetService <IContainerOperationProvider>();
                        var containers        = new List <Container.ContainerInfo>();
                        if (jobContext.Container != null)
                        {
                            containers.Add(jobContext.Container);
                        }
                        containers.AddRange(jobContext.ServiceContainers);

                        preJobSteps.Add(new JobExtensionRunner(runAsync: containerProvider.StartContainersAsync,
                                                               condition: $"{PipelineTemplateConstants.Success}()",
                                                               displayName: "Initialize containers",
                                                               data: (object)containers));
                    }

                    // Add action steps
                    foreach (var step in message.Steps)
                    {
                        if (step.Type == Pipelines.StepType.Action)
                        {
                            var action = step as Pipelines.ActionStep;
                            Trace.Info($"Adding {action.DisplayName}.");
                            var actionRunner = HostContext.CreateService <IActionRunner>();
                            actionRunner.Action    = action;
                            actionRunner.Stage     = ActionRunStage.Main;
                            actionRunner.Condition = step.Condition;
                            var contextData = new Pipelines.ContextData.DictionaryContextData();
                            if (message.ContextData?.Count > 0)
                            {
                                foreach (var pair in message.ContextData)
                                {
                                    contextData[pair.Key] = pair.Value;
                                }
                            }

                            actionRunner.TryEvaluateDisplayName(contextData, context);
                            jobSteps.Add(actionRunner);

                            if (prepareResult.PreStepTracker.TryGetValue(step.Id, out var preStep))
                            {
                                Trace.Info($"Adding pre-{action.DisplayName}.");
                                preStep.TryEvaluateDisplayName(contextData, context);
                                preStep.DisplayName = $"Pre {preStep.DisplayName}";
                                preJobSteps.Add(preStep);
                            }
                        }
                    }

                    var intraActionStates = new Dictionary <Guid, Dictionary <string, string> >();
                    foreach (var preStep in prepareResult.PreStepTracker)
                    {
                        intraActionStates[preStep.Key] = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
                    }

                    // Create execution context for pre-job steps
                    foreach (var step in preJobSteps)
                    {
                        if (step is JobExtensionRunner)
                        {
                            JobExtensionRunner extensionStep = step as JobExtensionRunner;
                            ArgUtil.NotNull(extensionStep, extensionStep.DisplayName);
                            Guid stepId = Guid.NewGuid();
                            extensionStep.ExecutionContext = jobContext.CreateChild(stepId, extensionStep.DisplayName, null, null, stepId.ToString("N"));
                        }
                        else if (step is IActionRunner actionStep)
                        {
                            ArgUtil.NotNull(actionStep, step.DisplayName);
                            Guid stepId = Guid.NewGuid();
                            actionStep.ExecutionContext = jobContext.CreateChild(stepId, actionStep.DisplayName, stepId.ToString("N"), null, null, intraActionStates[actionStep.Action.Id]);
                        }
                    }

                    // Create execution context for job steps
                    foreach (var step in jobSteps)
                    {
                        if (step is IActionRunner actionStep)
                        {
                            ArgUtil.NotNull(actionStep, step.DisplayName);
                            intraActionStates.TryGetValue(actionStep.Action.Id, out var intraActionState);
                            actionStep.ExecutionContext = jobContext.CreateChild(actionStep.Action.Id, actionStep.DisplayName, actionStep.Action.Name, actionStep.Action.ScopeName, actionStep.Action.ContextName, intraActionState);
                        }
                    }

                    List <IStep> steps = new List <IStep>();
                    steps.AddRange(preJobSteps);
                    steps.AddRange(jobSteps);

                    // Prepare for orphan process cleanup
                    _processCleanup = jobContext.Variables.GetBoolean("process.clean") ?? true;
                    if (_processCleanup)
                    {
                        // Set the RUNNER_TRACKING_ID env variable.
                        Environment.SetEnvironmentVariable(Constants.ProcessTrackingId, _processLookupId);
                        context.Debug("Collect running processes for tracking orphan processes.");

                        // Take a snapshot of current running processes
                        Dictionary <int, Process> processes = SnapshotProcesses();
                        foreach (var proc in processes)
                        {
                            // Pid_ProcessName
                            _existingProcesses.Add($"{proc.Key}_{proc.Value.ProcessName}");
                        }
                    }

                    return(steps);
                }
                catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
                {
                    // Log the exception and cancel the JobExtension Initialization.
                    Trace.Error($"Caught cancellation exception from JobExtension Initialization: {ex}");
                    context.Error(ex);
                    context.Result = TaskResult.Canceled;
                    throw;
                }
                catch (Exception ex)
                {
                    // Log the error and fail the JobExtension Initialization.
                    Trace.Error($"Caught exception from JobExtension Initialization: {ex}");
                    context.Error(ex);
                    context.Result = TaskResult.Failed;
                    throw;
                }
                finally
                {
                    context.Debug("Finishing: Set up job");
                    context.Complete();
                }
            }
        }
예제 #4
0
        private bool IsMessageIdentical(Pipelines.AgentJobRequestMessage source, Pipelines.AgentJobRequestMessage target)
        {
            if (source == null && target == null)
            {
                return(true);
            }
            if (source != null && target == null)
            {
                return(false);
            }
            if (source == null && target != null)
            {
                return(false);
            }
            if (JsonUtility.ToString(source.JobContainer) != JsonUtility.ToString(target.JobContainer))
            {
                return(false);
            }
            if (source.JobDisplayName != target.JobDisplayName)
            {
                return(false);
            }
            if (source.JobId != target.JobId)
            {
                return(false);
            }
            if (source.JobName != target.JobName)
            {
                return(false);
            }
            if (source.MaskHints.Count != target.MaskHints.Count)
            {
                return(false);
            }
            if (source.MessageType != target.MessageType)
            {
                return(false);
            }
            if (source.Plan.PlanId != target.Plan.PlanId)
            {
                return(false);
            }
            if (source.RequestId != target.RequestId)
            {
                return(false);
            }
            if (source.Resources.Endpoints.Count != target.Resources.Endpoints.Count)
            {
                return(false);
            }
            if (source.Steps.Count != target.Steps.Count)
            {
                return(false);
            }
            if (source.Variables.Count != target.Variables.Count)
            {
                return(false);
            }

            return(true);
        }
예제 #5
0
        private async Task RunAsync(Pipelines.AgentJobRequestMessage message, WorkerDispatcher previousJobDispatch, CancellationToken jobRequestCancellationToken, CancellationToken workerCancelTimeoutKillToken)
        {
            if (previousJobDispatch != null)
            {
                Trace.Verbose($"Make sure the previous job request {previousJobDispatch.JobId} has successfully finished on worker.");
                await EnsureDispatchFinished(previousJobDispatch);
            }
            else
            {
                Trace.Verbose($"This is the first job request.");
            }

            var term = HostContext.GetService <ITerminal>();

            term.WriteLine($"{DateTime.UtcNow:u}: Running job: {message.JobDisplayName}");

            // first job request renew succeed.
            TaskCompletionSource <int> firstJobRequestRenewed = new TaskCompletionSource <int>();
            var notification = HostContext.GetService <IJobNotification>();

            // lock renew cancellation token.
            using (var lockRenewalTokenSource = new CancellationTokenSource())
                using (var workerProcessCancelTokenSource = new CancellationTokenSource())
                {
                    long requestId = message.RequestId;
                    Guid lockToken = Guid.Empty; // lockToken has never been used, keep this here of compat

                    // start renew job request
                    Trace.Info($"Start renew job request {requestId} for job {message.JobId}.");
                    Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, firstJobRequestRenewed, lockRenewalTokenSource.Token);

                    // wait till first renew succeed or job request is canceled
                    // not even start worker if the first renew fail
                    await Task.WhenAny(firstJobRequestRenewed.Task, renewJobRequest, Task.Delay(-1, jobRequestCancellationToken));

                    if (renewJobRequest.IsCompleted)
                    {
                        // renew job request task complete means we run out of retry for the first job request renew.
                        Trace.Info($"Unable to renew job request for job {message.JobId} for the first time, stop dispatching job to worker.");
                        return;
                    }

                    if (jobRequestCancellationToken.IsCancellationRequested)
                    {
                        Trace.Info($"Stop renew job request for job {message.JobId}.");
                        // stop renew lock
                        lockRenewalTokenSource.Cancel();
                        // renew job request should never blows up.
                        await renewJobRequest;

                        // complete job request with result Cancelled
                        await CompleteJobRequestAsync(_poolId, message, lockToken, TaskResult.Canceled);

                        return;
                    }

                    HostContext.WritePerfCounter($"JobRequestRenewed_{requestId.ToString()}");

                    Task <int>    workerProcessTask = null;
                    object        _outputLock       = new object();
                    List <string> workerOutput      = new List <string>();
                    using (var processChannel = HostContext.CreateService <IProcessChannel>())
                        using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
                        {
                            // Start the process channel.
                            // It's OK if StartServer bubbles an execption after the worker process has already started.
                            // The worker will shutdown after 30 seconds if it hasn't received the job message.
                            processChannel.StartServer(
                                // Delegate to start the child process.
                                startProcess: (string pipeHandleOut, string pipeHandleIn) =>
                            {
                                // Validate args.
                                ArgUtil.NotNullOrEmpty(pipeHandleOut, nameof(pipeHandleOut));
                                ArgUtil.NotNullOrEmpty(pipeHandleIn, nameof(pipeHandleIn));

                                // Save STDOUT from worker, worker will use STDOUT report unhandle exception.
                                processInvoker.OutputDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stdout)
                                {
                                    if (!string.IsNullOrEmpty(stdout.Data))
                                    {
                                        lock (_outputLock)
                                        {
                                            workerOutput.Add(stdout.Data);
                                        }
                                    }
                                };

                                // Save STDERR from worker, worker will use STDERR on crash.
                                processInvoker.ErrorDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stderr)
                                {
                                    if (!string.IsNullOrEmpty(stderr.Data))
                                    {
                                        lock (_outputLock)
                                        {
                                            workerOutput.Add(stderr.Data);
                                        }
                                    }
                                };

                                // Start the child process.
                                HostContext.WritePerfCounter("StartingWorkerProcess");
                                var assemblyDirectory = HostContext.GetDirectory(WellKnownDirectory.Bin);
                                string workerFileName = Path.Combine(assemblyDirectory, _workerProcessName);
                                workerProcessTask     = processInvoker.ExecuteAsync(
                                    workingDirectory: assemblyDirectory,
                                    fileName: workerFileName,
                                    arguments: "spawnclient " + pipeHandleOut + " " + pipeHandleIn,
                                    environment: null,
                                    requireExitCodeZero: false,
                                    outputEncoding: null,
                                    killProcessOnCancel: true,
                                    redirectStandardIn: null,
                                    inheritConsoleHandler: false,
                                    keepStandardInOpen: false,
                                    highPriorityProcess: true,
                                    cancellationToken: workerProcessCancelTokenSource.Token);
                            });

                            // Send the job request message.
                            // Kill the worker process if sending the job message times out. The worker
                            // process may have successfully received the job message.
                            try
                            {
                                Trace.Info($"Send job request message to worker for job {message.JobId}.");
                                HostContext.WritePerfCounter($"RunnerSendingJobToWorker_{message.JobId}");
                                using (var csSendJobRequest = new CancellationTokenSource(_channelTimeout))
                                {
                                    await processChannel.SendAsync(
                                        messageType : MessageType.NewJobRequest,
                                        body : JsonUtility.ToString(message),
                                        cancellationToken : csSendJobRequest.Token);
                                }
                            }
                            catch (OperationCanceledException)
                            {
                                // message send been cancelled.
                                // timeout 30 sec. kill worker.
                                Trace.Info($"Job request message sending for job {message.JobId} been cancelled, kill running worker.");
                                workerProcessCancelTokenSource.Cancel();
                                try
                                {
                                    await workerProcessTask;
                                }
                                catch (OperationCanceledException)
                                {
                                    Trace.Info("worker process has been killed.");
                                }

                                Trace.Info($"Stop renew job request for job {message.JobId}.");
                                // stop renew lock
                                lockRenewalTokenSource.Cancel();
                                // renew job request should never blows up.
                                await renewJobRequest;

                                // not finish the job request since the job haven't run on worker at all, we will not going to set a result to server.
                                return;
                            }

                            // we get first jobrequest renew succeed and start the worker process with the job message.
                            // send notification to machine provisioner.
                            var systemConnection = message.Resources.Endpoints.SingleOrDefault(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
                            var accessToken      = systemConnection?.Authorization?.Parameters["AccessToken"];
                            notification.JobStarted(message.JobId, accessToken, systemConnection.Url);

                            HostContext.WritePerfCounter($"SentJobToWorker_{requestId.ToString()}");

                            try
                            {
                                TaskResult resultOnAbandonOrCancel = TaskResult.Succeeded;
                                // wait for renewlock, worker process or cancellation token been fired.
                                var completedTask = await Task.WhenAny(renewJobRequest, workerProcessTask, Task.Delay(-1, jobRequestCancellationToken));

                                if (completedTask == workerProcessTask)
                                {
                                    // worker finished successfully, complete job request with result, attach unhandled exception reported by worker, stop renew lock, job has finished.
                                    int returnCode = await workerProcessTask;
                                    Trace.Info($"Worker finished for job {message.JobId}. Code: " + returnCode);

                                    string detailInfo = null;
                                    if (!TaskResultUtil.IsValidReturnCode(returnCode))
                                    {
                                        detailInfo = string.Join(Environment.NewLine, workerOutput);
                                        Trace.Info($"Return code {returnCode} indicate worker encounter an unhandled exception or app crash, attach worker stdout/stderr to JobRequest result.");
                                        await LogWorkerProcessUnhandledException(message, detailInfo);
                                    }

                                    TaskResult result = TaskResultUtil.TranslateFromReturnCode(returnCode);
                                    Trace.Info($"finish job request for job {message.JobId} with result: {result}");
                                    term.WriteLine($"{DateTime.UtcNow:u}: Job {message.JobDisplayName} completed with result: {result}");

                                    Trace.Info($"Stop renew job request for job {message.JobId}.");
                                    // stop renew lock
                                    lockRenewalTokenSource.Cancel();
                                    // renew job request should never blows up.
                                    await renewJobRequest;

                                    // complete job request
                                    await CompleteJobRequestAsync(_poolId, message, lockToken, result, detailInfo);

                                    // print out unhandled exception happened in worker after we complete job request.
                                    // when we run out of disk space, report back to server has higher priority.
                                    if (!string.IsNullOrEmpty(detailInfo))
                                    {
                                        Trace.Error("Unhandled exception happened in worker:");
                                        Trace.Error(detailInfo);
                                    }

                                    return;
                                }
                                else if (completedTask == renewJobRequest)
                                {
                                    resultOnAbandonOrCancel = TaskResult.Abandoned;
                                }
                                else
                                {
                                    resultOnAbandonOrCancel = TaskResult.Canceled;
                                }

                                // renew job request completed or job request cancellation token been fired for RunAsync(jobrequestmessage)
                                // cancel worker gracefully first, then kill it after worker cancel timeout
                                try
                                {
                                    Trace.Info($"Send job cancellation message to worker for job {message.JobId}.");
                                    using (var csSendCancel = new CancellationTokenSource(_channelTimeout))
                                    {
                                        var messageType = MessageType.CancelRequest;
                                        if (HostContext.RunnerShutdownToken.IsCancellationRequested)
                                        {
                                            switch (HostContext.RunnerShutdownReason)
                                            {
                                            case ShutdownReason.UserCancelled:
                                                messageType = MessageType.RunnerShutdown;
                                                break;

                                            case ShutdownReason.OperatingSystemShutdown:
                                                messageType = MessageType.OperatingSystemShutdown;
                                                break;
                                            }
                                        }

                                        await processChannel.SendAsync(
                                            messageType : messageType,
                                            body : string.Empty,
                                            cancellationToken : csSendCancel.Token);
                                    }
                                }
                                catch (OperationCanceledException)
                                {
                                    // message send been cancelled.
                                    Trace.Info($"Job cancel message sending for job {message.JobId} been cancelled, kill running worker.");
                                    workerProcessCancelTokenSource.Cancel();
                                    try
                                    {
                                        await workerProcessTask;
                                    }
                                    catch (OperationCanceledException)
                                    {
                                        Trace.Info("worker process has been killed.");
                                    }
                                }

                                // wait worker to exit
                                // if worker doesn't exit within timeout, then kill worker.
                                completedTask = await Task.WhenAny(workerProcessTask, Task.Delay(-1, workerCancelTimeoutKillToken));

                                // worker haven't exit within cancellation timeout.
                                if (completedTask != workerProcessTask)
                                {
                                    Trace.Info($"worker process for job {message.JobId} haven't exit within cancellation timout, kill running worker.");
                                    workerProcessCancelTokenSource.Cancel();
                                    try
                                    {
                                        await workerProcessTask;
                                    }
                                    catch (OperationCanceledException)
                                    {
                                        Trace.Info("worker process has been killed.");
                                    }
                                }

                                Trace.Info($"finish job request for job {message.JobId} with result: {resultOnAbandonOrCancel}");
                                term.WriteLine($"{DateTime.UtcNow:u}: Job {message.JobDisplayName} completed with result: {resultOnAbandonOrCancel}");
                                // complete job request with cancel result, stop renew lock, job has finished.

                                Trace.Info($"Stop renew job request for job {message.JobId}.");
                                // stop renew lock
                                lockRenewalTokenSource.Cancel();
                                // renew job request should never blows up.
                                await renewJobRequest;

                                // complete job request
                                await CompleteJobRequestAsync(_poolId, message, lockToken, resultOnAbandonOrCancel);
                            }
                            finally
                            {
                                // This should be the last thing to run so we don't notify external parties until actually finished
                                await notification.JobCompleted(message.JobId);
                            }
                        }
                }
        }
예제 #6
0
        private async Task <TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, TaskResult?taskResult = null)
        {
            jobContext.Debug($"Finishing: {message.JobDisplayName}");
            TaskResult result = jobContext.Complete(taskResult);

            try
            {
                await ShutdownQueue(throwOnFailure : true);
            }
            catch (Exception ex)
            {
                Trace.Error($"Caught exception from {nameof(JobServerQueue)}.{nameof(_jobServerQueue.ShutdownAsync)}");
                Trace.Error("This indicate a failure during publish output variables. Fail the job to prevent unexpected job outputs.");
                Trace.Error(ex);
                result = TaskResultUtil.MergeTaskResults(result, TaskResult.Failed);
            }

            // Clean TEMP after finish process jobserverqueue, since there might be a pending fileupload still use the TEMP dir.
            _tempDirectoryManager?.CleanupTempDirectory();

            if (!jobContext.Features.HasFlag(PlanFeatures.JobCompletedPlanEvent))
            {
                Trace.Info($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}");
                return(result);
            }

            Trace.Info("Raising job completed event.");
            var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result);

            var completeJobRetryLimit = 5;
            var exceptions            = new List <Exception>();

            while (completeJobRetryLimit-- > 0)
            {
                try
                {
                    await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, default(CancellationToken));

                    return(result);
                }
                catch (TaskOrchestrationPlanNotFoundException ex)
                {
                    Trace.Error($"TaskOrchestrationPlanNotFoundException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
                    Trace.Error(ex);
                    return(TaskResult.Failed);
                }
                catch (TaskOrchestrationPlanSecurityException ex)
                {
                    Trace.Error($"TaskOrchestrationPlanSecurityException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
                    Trace.Error(ex);
                    return(TaskResult.Failed);
                }
                catch (Exception ex)
                {
                    Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}.");
                    Trace.Error(ex);
                    exceptions.Add(ex);
                }

                // delay 5 seconds before next retry.
                await Task.Delay(TimeSpan.FromSeconds(5));
            }

            // rethrow exceptions from all attempts.
            throw new AggregateException(exceptions);
        }
예제 #7
0
        public async Task <TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken)
        {
            // Validate parameters.
            Trace.Entering();
            ArgUtil.NotNull(message, nameof(message));
            ArgUtil.NotNull(message.Resources, nameof(message.Resources));
            ArgUtil.NotNull(message.Variables, nameof(message.Variables));
            ArgUtil.NotNull(message.Steps, nameof(message.Steps));
            Trace.Info("Job ID {0}", message.JobId);

            DateTime jobStartTimeUtc = DateTime.UtcNow;

            ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));

            // Setup the job server and job server queue.
            var            jobServer           = HostContext.GetService <IJobServer>();
            VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection);
            Uri            jobServerUrl        = systemConnection.Url;

            Trace.Info($"Creating job server with URL: {jobServerUrl}");
            // jobServerQueue is the throttling reporter.
            _jobServerQueue = HostContext.GetService <IJobServerQueue>();
            VssConnection jobConnection = VssUtil.CreateConnection(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(_jobServerQueue) });
            await jobServer.ConnectAsync(jobConnection);

            _jobServerQueue.Start(message);
            HostContext.WritePerfCounter($"WorkerJobServerQueueStarted_{message.RequestId.ToString()}");

            IExecutionContext             jobContext = null;
            CancellationTokenRegistration?runnerShutdownRegistration = null;

            try
            {
                // Create the job execution context.
                jobContext = HostContext.CreateService <IExecutionContext>();
                jobContext.InitializeJob(message, jobRequestCancellationToken);
                Trace.Info("Starting the job execution context.");
                jobContext.Start();
                jobContext.Debug($"Starting: {message.JobDisplayName}");

                runnerShutdownRegistration = HostContext.RunnerShutdownToken.Register(() =>
                {
                    // log an issue, then runner get shutdown by Ctrl-C or Ctrl-Break.
                    // the server will use Ctrl-Break to tells the runner that operating system is shutting down.
                    string errorMessage;
                    switch (HostContext.RunnerShutdownReason)
                    {
                    case ShutdownReason.UserCancelled:
                        errorMessage = "The runner has received a shutdown signal. This can happen when the runner service is stopped, or a manually started runner is canceled.";
                        break;

                    case ShutdownReason.OperatingSystemShutdown:
                        errorMessage = $"Operating system is shutting down for computer '{Environment.MachineName}'";
                        break;

                    default:
                        throw new ArgumentException(HostContext.RunnerShutdownReason.ToString(), nameof(HostContext.RunnerShutdownReason));
                    }
                    jobContext.AddIssue(new Issue()
                    {
                        Type = IssueType.Error, Message = errorMessage
                    });
                });

                // Validate directory permissions.
                string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
                Trace.Info($"Validating directory permissions for: '{workDirectory}'");
                try
                {
                    Directory.CreateDirectory(workDirectory);
                    IOUtil.ValidateExecutePermission(workDirectory);
                }
                catch (Exception ex)
                {
                    Trace.Error(ex);
                    jobContext.Error(ex);
                    return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
                }

                if (jobContext.WriteDebug)
                {
                    jobContext.SetRunnerContext("debug", "1");
                }

                jobContext.SetRunnerContext("os", VarUtil.OS);

                string toolsDirectory = HostContext.GetDirectory(WellKnownDirectory.Tools);
                Directory.CreateDirectory(toolsDirectory);
                jobContext.SetRunnerContext("tool_cache", toolsDirectory);

                // Setup TEMP directories
                _tempDirectoryManager = HostContext.GetService <ITempDirectoryManager>();
                _tempDirectoryManager.InitializeTempDirectory(jobContext);

                // // Expand container properties
                // jobContext.Container?.ExpandProperties(jobContext.Variables);
                // foreach (var sidecar in jobContext.SidecarContainers)
                // {
                //     sidecar.ExpandProperties(jobContext.Variables);
                // }

                // Get the job extension.
                Trace.Info("Getting job extension.");
                IJobExtension jobExtension = HostContext.CreateService <IJobExtension>();
                List <IStep>  jobSteps     = null;
                try
                {
                    Trace.Info("Initialize job. Getting all job steps.");
                    jobSteps = await jobExtension.InitializeJob(jobContext, message);
                }
                catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
                {
                    // set the job to canceled
                    // don't log error issue to job ExecutionContext, since server owns the job level issue
                    Trace.Error($"Job is canceled during initialize.");
                    Trace.Error($"Caught exception: {ex}");
                    return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled));
                }
                catch (Exception ex)
                {
                    // set the job to failed.
                    // don't log error issue to job ExecutionContext, since server owns the job level issue
                    Trace.Error($"Job initialize failed.");
                    Trace.Error($"Caught exception from {nameof(jobExtension.InitializeJob)}: {ex}");
                    return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
                }

                // trace out all steps
                Trace.Info($"Total job steps: {jobSteps.Count}.");
                Trace.Verbose($"Job steps: '{string.Join(", ", jobSteps.Select(x => x.DisplayName))}'");
                HostContext.WritePerfCounter($"WorkerJobInitialized_{message.RequestId.ToString()}");

                // Run all job steps
                Trace.Info("Run all job steps.");
                var stepsRunner = HostContext.GetService <IStepsRunner>();
                try
                {
                    foreach (var step in jobSteps)
                    {
                        jobContext.JobSteps.Enqueue(step);
                    }

                    await stepsRunner.RunAsync(jobContext);
                }
                catch (Exception ex)
                {
                    // StepRunner should never throw exception out.
                    // End up here mean there is a bug in StepRunner
                    // Log the error and fail the job.
                    Trace.Error($"Caught exception from job steps {nameof(StepsRunner)}: {ex}");
                    jobContext.Error(ex);
                    return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
                }
                finally
                {
                    Trace.Info("Finalize job.");
                    jobExtension.FinalizeJob(jobContext, message, jobStartTimeUtc);
                }

                Trace.Info($"Job result after all job steps finish: {jobContext.Result ?? TaskResult.Succeeded}");

                Trace.Info("Completing the job execution context.");
                return(await CompleteJobAsync(jobServer, jobContext, message));
            }
            finally
            {
                if (runnerShutdownRegistration != null)
                {
                    runnerShutdownRegistration.Value.Dispose();
                    runnerShutdownRegistration = null;
                }

                await ShutdownQueue(throwOnFailure : false);
            }
        }
예제 #8
0
        public void RegisterPostJobAction_NotRegisterPostTwice()
        {
            using (TestHostContext hc = CreateTestContext())
            {
                // Arrange: Create a job request message.
                TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
                TimelineReference timeline          = new TimelineReference();
                Guid   jobId      = Guid.NewGuid();
                string jobName    = "some job name";
                var    jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary <string, VariableValue>(), new List <MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List <Pipelines.ActionStep>(), null, null, null, null);
                jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                {
                    Alias   = Pipelines.PipelineConstants.SelfAlias,
                    Id      = "github",
                    Version = "sha1"
                });
                jobRequest.ContextData["github"]           = new Pipelines.ContextData.DictionaryContextData();
                jobRequest.Variables["ACTIONS_STEP_DEBUG"] = "true";

                // Arrange: Setup the paging logger.
                var pagingLogger1  = new Mock <IPagingLogger>();
                var pagingLogger2  = new Mock <IPagingLogger>();
                var pagingLogger3  = new Mock <IPagingLogger>();
                var pagingLogger4  = new Mock <IPagingLogger>();
                var pagingLogger5  = new Mock <IPagingLogger>();
                var jobServerQueue = new Mock <IJobServerQueue>();
                jobServerQueue.Setup(x => x.QueueTimelineRecordUpdate(It.IsAny <Guid>(), It.IsAny <TimelineRecord>()));
                jobServerQueue.Setup(x => x.QueueWebConsoleLine(It.IsAny <Guid>(), It.IsAny <string>(), It.IsAny <long?>())).Callback((Guid id, string msg, long?lineNumber) => { hc.GetTrace().Info(msg); });

                var actionRunner1 = new ActionRunner();
                actionRunner1.Initialize(hc);
                var actionRunner2 = new ActionRunner();
                actionRunner2.Initialize(hc);

                hc.EnqueueInstance(pagingLogger1.Object);
                hc.EnqueueInstance(pagingLogger2.Object);
                hc.EnqueueInstance(pagingLogger3.Object);
                hc.EnqueueInstance(pagingLogger4.Object);
                hc.EnqueueInstance(pagingLogger5.Object);
                hc.EnqueueInstance(actionRunner1 as IActionRunner);
                hc.EnqueueInstance(actionRunner2 as IActionRunner);
                hc.SetSingleton(jobServerQueue.Object);

                var jobContext = new Runner.Worker.ExecutionContext();
                jobContext.Initialize(hc);

                // Act.
                jobContext.InitializeJob(jobRequest, CancellationToken.None);

                var action1 = jobContext.CreateChild(Guid.NewGuid(), "action_1_pre", "action_1_pre", null, null, 0);
                var action2 = jobContext.CreateChild(Guid.NewGuid(), "action_1_main", "action_1_main", null, null, 0);

                var actionId    = Guid.NewGuid();
                var postRunner1 = hc.CreateService <IActionRunner>();
                postRunner1.Action = new Pipelines.ActionStep()
                {
                    Id = actionId, Name = "post1", DisplayName = "Test 1", Reference = new Pipelines.RepositoryPathReference()
                    {
                        Name = "actions/action"
                    }
                };
                postRunner1.Stage       = ActionRunStage.Post;
                postRunner1.Condition   = "always()";
                postRunner1.DisplayName = "post1";


                var postRunner2 = hc.CreateService <IActionRunner>();
                postRunner2.Action = new Pipelines.ActionStep()
                {
                    Id = actionId, Name = "post2", DisplayName = "Test 2", Reference = new Pipelines.RepositoryPathReference()
                    {
                        Name = "actions/action"
                    }
                };
                postRunner2.Stage       = ActionRunStage.Post;
                postRunner2.Condition   = "always()";
                postRunner2.DisplayName = "post2";

                action1.RegisterPostJobStep(postRunner1);
                action2.RegisterPostJobStep(postRunner2);

                Assert.NotNull(jobContext.JobSteps);
                Assert.NotNull(jobContext.PostJobSteps);
                Assert.Equal(1, jobContext.PostJobSteps.Count);
                var post1 = jobContext.PostJobSteps.Pop();

                Assert.Equal("post1", (post1 as IActionRunner).Action.Name);

                Assert.Equal(ActionRunStage.Post, (post1 as IActionRunner).Stage);

                Assert.Equal("always()", (post1 as IActionRunner).Condition);
            }
        }
예제 #9
0
        public void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token)
        {
            // Validation
            Trace.Entering();
            ArgUtil.NotNull(message, nameof(message));
            ArgUtil.NotNull(message.Resources, nameof(message.Resources));
            ArgUtil.NotNull(message.Variables, nameof(message.Variables));
            ArgUtil.NotNull(message.Plan, nameof(message.Plan));

            _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token);

            // Features
            Features = PlanUtil.GetFeatures(message.Plan);

            // Endpoints
            Endpoints = message.Resources.Endpoints;

            // Variables
            Variables = new Variables(HostContext, message.Variables);

            // Environment variables shared across all actions
            EnvironmentVariables = new Dictionary <string, string>(VarUtil.EnvironmentVariableKeyComparer);

            // Job defaults shared across all actions
            JobDefaults = new Dictionary <string, IDictionary <string, string> >(StringComparer.OrdinalIgnoreCase);

            // Job Outputs
            JobOutputs = new Dictionary <string, VariableValue>(StringComparer.OrdinalIgnoreCase);

            // Service container info
            ServiceContainers = new List <ContainerInfo>();

            // Steps context (StepsRunner manages adding the scoped steps context)
            StepsContext = new StepsContext();

            // Scopes
            Scopes = new Dictionary <String, ContextScope>(StringComparer.OrdinalIgnoreCase);
            if (message.Scopes?.Count > 0)
            {
                foreach (var scope in message.Scopes)
                {
                    Scopes[scope.Name] = scope;
                }
            }

            // File table
            FileTable = new List <String>(message.FileTable ?? new string[0]);

            // Expression values
            if (message.ContextData?.Count > 0)
            {
                foreach (var pair in message.ContextData)
                {
                    ExpressionValues[pair.Key] = pair.Value;
                }
            }

            ExpressionValues["secrets"] = Variables.ToSecretsContext();
            ExpressionValues["runner"]  = new RunnerContext();
            ExpressionValues["job"]     = new JobContext();

            Trace.Info("Initialize GitHub context");
            var githubAccessToken  = new StringContextData(Variables.Get("system.github.token"));
            var base64EncodedToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{githubAccessToken}"));

            HostContext.SecretMasker.AddValue(base64EncodedToken);
            var githubJob     = Variables.Get("system.github.job");
            var githubContext = new GitHubContext();

            githubContext["token"] = githubAccessToken;
            if (!string.IsNullOrEmpty(githubJob))
            {
                githubContext["job"] = new StringContextData(githubJob);
            }
            var githubDictionary = ExpressionValues["github"].AssertDictionary("github");

            foreach (var pair in githubDictionary)
            {
                githubContext[pair.Key] = pair.Value;
            }
            ExpressionValues["github"] = githubContext;

            Trace.Info("Initialize Env context");
#if OS_WINDOWS
            ExpressionValues["env"] = new DictionaryContextData();
#else
            ExpressionValues["env"] = new CaseSensitiveDictionaryContextData();
#endif

            // Prepend Path
            PrependPath = new List <string>();

            // JobSteps for job ExecutionContext
            JobSteps = new Queue <IStep>();

            // PostJobSteps for job ExecutionContext
            PostJobSteps = new Stack <IStep>();

            // Job timeline record.
            InitializeTimelineRecord(
                timelineId: message.Timeline.Id,
                timelineRecordId: message.JobId,
                parentTimelineRecordId: null,
                recordType: ExecutionContextType.Job,
                displayName: message.JobDisplayName,
                refName: message.JobName,
                order: null); // The job timeline record's order is set by server.

            // Logger (must be initialized before writing warnings).
            _logger = HostContext.CreateService <IPagingLogger>();
            _logger.Setup(_mainTimelineId, _record.Id);

            // Initialize 'echo on action command success' property, default to false, unless Step_Debug is set
            EchoOnActionCommand = Variables.Step_Debug ?? false;

            // Verbosity (from GitHub.Step_Debug).
            WriteDebug = Variables.Step_Debug ?? false;

            // Hook up JobServerQueueThrottling event, we will log warning on server tarpit.
            _jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived;
        }
예제 #10
0
        public void UploadDiagnosticLogs(IExecutionContext executionContext,
                                         IExecutionContext parentContext,
                                         Pipelines.AgentJobRequestMessage message,
                                         DateTime jobStartTimeUtc)
        {
            executionContext.Debug("Starting diagnostic file upload.");

            // Setup folders
            // \_layout\_work\_temp\[jobname-support]
            executionContext.Debug("Setting up diagnostic log folders.");
            string tempDirectory = HostContext.GetDirectory(WellKnownDirectory.Temp);

            ArgUtil.Directory(tempDirectory, nameof(tempDirectory));

            string supportRootFolder = Path.Combine(tempDirectory, message.JobName + "-support");

            Directory.CreateDirectory(supportRootFolder);

            // \_layout\_work\_temp\[jobname-support]\files
            executionContext.Debug("Creating diagnostic log files folder.");
            string supportFilesFolder = Path.Combine(supportRootFolder, "files");

            Directory.CreateDirectory(supportFilesFolder);

            // Create the environment file
            // \_layout\_work\_temp\[jobname-support]\files\environment.txt
            var            configurationStore = HostContext.GetService <IConfigurationStore>();
            RunnerSettings settings           = configurationStore.GetSettings();
            int            runnerId           = settings.AgentId;
            string         runnerName         = settings.AgentName;
            int            poolId             = settings.PoolId;

            // Copy worker diagnostic log files
            List <string> workerDiagnosticLogFiles = GetWorkerDiagnosticLogFiles(HostContext.GetDirectory(WellKnownDirectory.Diag), jobStartTimeUtc);

            executionContext.Debug($"Copying {workerDiagnosticLogFiles.Count()} worker diagnostic logs.");

            foreach (string workerLogFile in workerDiagnosticLogFiles)
            {
                ArgUtil.File(workerLogFile, nameof(workerLogFile));

                string destination = Path.Combine(supportFilesFolder, Path.GetFileName(workerLogFile));
                File.Copy(workerLogFile, destination);
            }

            // Copy runner diag log files
            List <string> runnerDiagnosticLogFiles = GetRunnerDiagnosticLogFiles(HostContext.GetDirectory(WellKnownDirectory.Diag), jobStartTimeUtc);

            executionContext.Debug($"Copying {runnerDiagnosticLogFiles.Count()} runner diagnostic logs.");

            foreach (string runnerLogFile in runnerDiagnosticLogFiles)
            {
                ArgUtil.File(runnerLogFile, nameof(runnerLogFile));

                string destination = Path.Combine(supportFilesFolder, Path.GetFileName(runnerLogFile));
                File.Copy(runnerLogFile, destination);
            }

            executionContext.Debug("Zipping diagnostic files.");

            string buildNumber = executionContext.Variables.Build_Number ?? "UnknownBuildNumber";
            string buildName   = $"Build {buildNumber}";
            string phaseName   = executionContext.Variables.System_PhaseDisplayName ?? "UnknownPhaseName";

            // zip the files
            string diagnosticsZipFileName = $"{buildName}-{phaseName}.zip";
            string diagnosticsZipFilePath = Path.Combine(supportRootFolder, diagnosticsZipFileName);

            ZipFile.CreateFromDirectory(supportFilesFolder, diagnosticsZipFilePath);

            // upload the json metadata file
            executionContext.Debug("Uploading diagnostic metadata file.");
            string metadataFileName = $"diagnostics-{buildName}-{phaseName}.json";
            string metadataFilePath = Path.Combine(supportFilesFolder, metadataFileName);
            string phaseResult      = GetTaskResultAsString(executionContext.Result);

            IOUtil.SaveObject(new DiagnosticLogMetadata(runnerName, runnerId, poolId, phaseName, diagnosticsZipFileName, phaseResult), metadataFilePath);

            // TODO: Remove the parentContext Parameter and replace this with executioncontext. Currently a bug exists where these files do not upload correctly using that context.
            parentContext.QueueAttachFile(type: CoreAttachmentType.DiagnosticLog, name: metadataFileName, filePath: metadataFilePath);

            parentContext.QueueAttachFile(type: CoreAttachmentType.DiagnosticLog, name: diagnosticsZipFileName, filePath: diagnosticsZipFilePath);

            executionContext.Debug("Diagnostic file upload complete.");
        }
예제 #11
0
        private TestHostContext CreateTestContext([CallerMemberName] String testName = "")
        {
            var hc = new TestHostContext(this, testName);

            _jobEc                = new Runner.Worker.ExecutionContext();
            _config               = new Mock <IConfigurationStore>();
            _extensions           = new Mock <IExtensionManager>();
            _jobExtension         = new Mock <IJobExtension>();
            _jobServer            = new Mock <IJobServer>();
            _jobServerQueue       = new Mock <IJobServerQueue>();
            _stepRunner           = new Mock <IStepsRunner>();
            _logger               = new Mock <IPagingLogger>();
            _temp                 = new Mock <ITempDirectoryManager>();
            _diagnosticLogManager = new Mock <IDiagnosticLogManager>();

            if (_tokenSource != null)
            {
                _tokenSource.Dispose();
                _tokenSource = null;
            }

            _tokenSource = new CancellationTokenSource();
            var expressionManager = new ExpressionManager();

            expressionManager.Initialize(hc);
            hc.SetSingleton <IExpressionManager>(expressionManager);

            _jobRunner = new JobRunner();
            _jobRunner.Initialize(hc);

            TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
            TimelineReference timeline          = new Timeline(Guid.NewGuid());
            Guid jobId = Guid.NewGuid();

            _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, testName, testName, null, null, null, new Dictionary <string, VariableValue>(), new List <MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List <Pipelines.ActionStep>(), null);
            _message.Variables[Constants.Variables.System.Culture] = "en-US";
            _message.Resources.Endpoints.Add(new ServiceEndpoint()
            {
                Name          = WellKnownServiceEndpointNames.SystemVssConnection,
                Url           = new Uri("https://pipelines.actions.githubusercontent.com"),
                Authorization = new EndpointAuthorization()
                {
                    Scheme     = "Test",
                    Parameters =
                    {
                        { "AccessToken", "token" }
                    }
                },
            });

            _message.Resources.Repositories.Add(new Pipelines.RepositoryResource()
            {
                Alias   = Pipelines.PipelineConstants.SelfAlias,
                Id      = "github",
                Version = "sha1"
            });
            _message.ContextData.Add("github", new Pipelines.ContextData.DictionaryContextData());

            _initResult.Clear();

            _jobExtension.Setup(x => x.InitializeJob(It.IsAny <IExecutionContext>(), It.IsAny <Pipelines.AgentJobRequestMessage>())).
            Returns(Task.FromResult(_initResult));

            var settings = new RunnerSettings
            {
                AgentId    = 1,
                AgentName  = "agent1",
                ServerUrl  = "https://pipelines.actions.githubusercontent.com",
                WorkFolder = "_work",
            };

            _config.Setup(x => x.GetSettings())
            .Returns(settings);

            _logger.Setup(x => x.Setup(It.IsAny <Guid>(), It.IsAny <Guid>()));

            hc.SetSingleton(_config.Object);
            hc.SetSingleton(_jobServer.Object);
            hc.SetSingleton(_jobServerQueue.Object);
            hc.SetSingleton(_stepRunner.Object);
            hc.SetSingleton(_extensions.Object);
            hc.SetSingleton(_temp.Object);
            hc.SetSingleton(_diagnosticLogManager.Object);
            hc.EnqueueInstance <IExecutionContext>(_jobEc);
            hc.EnqueueInstance <IPagingLogger>(_logger.Object);
            hc.EnqueueInstance <IJobExtension>(_jobExtension.Object);
            return(hc);
        }
예제 #12
0
        public async Task <TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken)
        {
            // Validate parameters.
            Trace.Entering();
            ArgUtil.NotNull(message, nameof(message));
            ArgUtil.NotNull(message.Resources, nameof(message.Resources));
            ArgUtil.NotNull(message.Variables, nameof(message.Variables));
            ArgUtil.NotNull(message.Steps, nameof(message.Steps));
            Trace.Info("Job ID {0}", message.JobId);

            DateTime jobStartTimeUtc = DateTime.UtcNow;

            ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));

            // Setup the job server and job server queue.
            var            jobServer           = HostContext.GetService <IJobServer>();
            VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection);
            Uri            jobServerUrl        = systemConnection.Url;

            Trace.Info($"Creating job server with URL: {jobServerUrl}");
            // jobServerQueue is the throttling reporter.
            _jobServerQueue = HostContext.GetService <IJobServerQueue>();
            VssConnection jobConnection = VssUtil.CreateConnection(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(_jobServerQueue) });
            await jobServer.ConnectAsync(jobConnection);

            _jobServerQueue.Start(message);
            HostContext.WritePerfCounter($"WorkerJobServerQueueStarted_{message.RequestId.ToString()}");

            IExecutionContext             jobContext = null;
            CancellationTokenRegistration?runnerShutdownRegistration = null;

            try
            {
                // Create the job execution context.
                jobContext = HostContext.CreateService <IExecutionContext>();
                jobContext.InitializeJob(message, jobRequestCancellationToken);
                Trace.Info("Starting the job execution context.");
                jobContext.Start();
                jobContext.Debug($"Starting: {message.JobDisplayName}");

                // RUST: If the event type is not allowed exit the job before anything is run.
                var rustExpectedEvent = System.Environment.GetEnvironmentVariable("RUST_WHITELISTED_EVENT_NAME");
                if (rustExpectedEvent != null)
                {
                    var rustGitHubContext = (Pipelines.ContextData.DictionaryContextData)message.ContextData["github"];
                    var rustEventName     = rustGitHubContext["event_name"].ToString();
                    if (rustEventName != rustExpectedEvent)
                    {
                        return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled));
                    }
                }

                runnerShutdownRegistration = HostContext.RunnerShutdownToken.Register(() =>
                {
                    // log an issue, then runner get shutdown by Ctrl-C or Ctrl-Break.
                    // the server will use Ctrl-Break to tells the runner that operating system is shutting down.
                    string errorMessage;
                    switch (HostContext.RunnerShutdownReason)
                    {
                    case ShutdownReason.UserCancelled:
                        errorMessage = "The runner has received a shutdown signal. This can happen when the runner service is stopped, or a manually started runner is canceled.";
                        break;

                    case ShutdownReason.OperatingSystemShutdown:
                        errorMessage = $"Operating system is shutting down for computer '{Environment.MachineName}'";
                        break;

                    default:
                        throw new ArgumentException(HostContext.RunnerShutdownReason.ToString(), nameof(HostContext.RunnerShutdownReason));
                    }
                    jobContext.AddIssue(new Issue()
                    {
                        Type = IssueType.Error, Message = errorMessage
                    });
                });

                // Validate directory permissions.
                string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
                Trace.Info($"Validating directory permissions for: '{workDirectory}'");
                try
                {
                    Directory.CreateDirectory(workDirectory);
                    IOUtil.ValidateExecutePermission(workDirectory);
                }
                catch (Exception ex)
                {
                    Trace.Error(ex);
                    jobContext.Error(ex);
                    return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
                }

                if (jobContext.Global.WriteDebug)
                {
                    jobContext.SetRunnerContext("debug", "1");
                }

                jobContext.SetRunnerContext("os", VarUtil.OS);
                jobContext.SetRunnerContext("arch", VarUtil.OSArchitecture);

                var runnerSettings = HostContext.GetService <IConfigurationStore>().GetSettings();
                jobContext.SetRunnerContext("name", runnerSettings.AgentName);

                string toolsDirectory = HostContext.GetDirectory(WellKnownDirectory.Tools);
                Directory.CreateDirectory(toolsDirectory);
                jobContext.SetRunnerContext("tool_cache", toolsDirectory);

                // Setup TEMP directories
                _tempDirectoryManager = HostContext.GetService <ITempDirectoryManager>();
                _tempDirectoryManager.InitializeTempDirectory(jobContext);

                // Get the job extension.
                Trace.Info("Getting job extension.");
                IJobExtension jobExtension = HostContext.CreateService <IJobExtension>();
                List <IStep>  jobSteps     = null;
                try
                {
                    Trace.Info("Initialize job. Getting all job steps.");
                    jobSteps = await jobExtension.InitializeJob(jobContext, message);
                }
                catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
                {
                    // set the job to canceled
                    // don't log error issue to job ExecutionContext, since server owns the job level issue
                    Trace.Error($"Job is canceled during initialize.");
                    Trace.Error($"Caught exception: {ex}");
                    return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled));
                }
                catch (Exception ex)
                {
                    // set the job to failed.
                    // don't log error issue to job ExecutionContext, since server owns the job level issue
                    Trace.Error($"Job initialize failed.");
                    Trace.Error($"Caught exception from {nameof(jobExtension.InitializeJob)}: {ex}");
                    return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
                }

                // trace out all steps
                Trace.Info($"Total job steps: {jobSteps.Count}.");
                Trace.Verbose($"Job steps: '{string.Join(", ", jobSteps.Select(x => x.DisplayName))}'");
                HostContext.WritePerfCounter($"WorkerJobInitialized_{message.RequestId.ToString()}");

                if (systemConnection.Data.TryGetValue("GenerateIdTokenUrl", out var generateIdTokenUrl) &&
                    !string.IsNullOrEmpty(generateIdTokenUrl))
                {
                    // Server won't issue ID_TOKEN for non-inprogress job.
                    // If the job is trying to use OIDC feature, we want the job to be marked as in-progress before running any customer's steps as much as we can.
                    // Timeline record update background process runs every 500ms, so delay 1000ms is enough for most of the cases
                    Trace.Info($"Waiting for job to be marked as started.");
                    await Task.WhenAny(_jobServerQueue.JobRecordUpdated.Task, Task.Delay(1000));
                }

                // Run all job steps
                Trace.Info("Run all job steps.");
                var stepsRunner = HostContext.GetService <IStepsRunner>();
                try
                {
                    foreach (var step in jobSteps)
                    {
                        jobContext.JobSteps.Enqueue(step);
                    }

                    await stepsRunner.RunAsync(jobContext);
                }
                catch (Exception ex)
                {
                    // StepRunner should never throw exception out.
                    // End up here mean there is a bug in StepRunner
                    // Log the error and fail the job.
                    Trace.Error($"Caught exception from job steps {nameof(StepsRunner)}: {ex}");
                    jobContext.Error(ex);
                    return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
                }
                finally
                {
                    Trace.Info("Finalize job.");
                    jobExtension.FinalizeJob(jobContext, message, jobStartTimeUtc);
                }

                Trace.Info($"Job result after all job steps finish: {jobContext.Result ?? TaskResult.Succeeded}");

                Trace.Info("Completing the job execution context.");
                return(await CompleteJobAsync(jobServer, jobContext, message));
            }
            finally
            {
                if (runnerShutdownRegistration != null)
                {
                    runnerShutdownRegistration.Value.Dispose();
                    runnerShutdownRegistration = null;
                }

                await ShutdownQueue(throwOnFailure : false);
            }
        }
예제 #13
0
        private TestHostContext CreateTestContext([CallerMemberName] String testName = "")
        {
            var hc = new TestHostContext(this, testName);

            _jobEc                = new Runner.Worker.ExecutionContext();
            _actionManager        = new Mock <IActionManager>();
            _jobServerQueue       = new Mock <IJobServerQueue>();
            _config               = new Mock <IConfigurationStore>();
            _logger               = new Mock <IPagingLogger>();
            _containerProvider    = new Mock <IContainerOperationProvider>();
            _diagnosticLogManager = new Mock <IDiagnosticLogManager>();
            _directoryManager     = new Mock <IPipelineDirectoryManager>();
            _directoryManager.Setup(x => x.PrepareDirectory(It.IsAny <IExecutionContext>(), It.IsAny <Pipelines.WorkspaceOptions>()))
            .Returns(new TrackingConfig()
            {
                PipelineDirectory = "runner", WorkspaceDirectory = "runner/runner"
            });

            IActionRunner step1 = new ActionRunner();
            IActionRunner step2 = new ActionRunner();
            IActionRunner step3 = new ActionRunner();
            IActionRunner step4 = new ActionRunner();
            IActionRunner step5 = new ActionRunner();

            _logger.Setup(x => x.Setup(It.IsAny <Guid>(), It.IsAny <Guid>()));
            var settings = new RunnerSettings
            {
                AgentId    = 1,
                AgentName  = "runner",
                ServerUrl  = "https://pipelines.actions.githubusercontent.com/abcd",
                WorkFolder = "_work",
            };

            _config.Setup(x => x.GetSettings())
            .Returns(settings);

            if (_tokenSource != null)
            {
                _tokenSource.Dispose();
                _tokenSource = null;
            }

            _tokenSource = new CancellationTokenSource();
            TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
            TimelineReference timeline          = new Timeline(Guid.NewGuid());

            List <Pipelines.ActionStep> steps = new List <Pipelines.ActionStep>()
            {
                new Pipelines.ActionStep()
                {
                    Id          = Guid.NewGuid(),
                    DisplayName = "action1",
                },
                new Pipelines.ActionStep()
                {
                    Id          = Guid.NewGuid(),
                    DisplayName = "action2",
                },
                new Pipelines.ActionStep()
                {
                    Id          = Guid.NewGuid(),
                    DisplayName = "action3",
                },
                new Pipelines.ActionStep()
                {
                    Id          = Guid.NewGuid(),
                    DisplayName = "action4",
                },
                new Pipelines.ActionStep()
                {
                    Id          = Guid.NewGuid(),
                    DisplayName = "action5",
                }
            };

            Guid jobId = Guid.NewGuid();

            _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary <string, VariableValue>(), new List <MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), steps, null, null, null, null);
            GitHubContext github = new GitHubContext();

            github["repository"] = new Pipelines.ContextData.StringContextData("actions/runner");
            _message.ContextData.Add("github", github);

            hc.SetSingleton(_actionManager.Object);
            hc.SetSingleton(_config.Object);
            hc.SetSingleton(_jobServerQueue.Object);
            hc.SetSingleton(_containerProvider.Object);
            hc.SetSingleton(_directoryManager.Object);
            hc.SetSingleton(_diagnosticLogManager.Object);
            hc.EnqueueInstance <IPagingLogger>(_logger.Object); // JobExecutionContext
            hc.EnqueueInstance <IPagingLogger>(_logger.Object); // Initial Job
            hc.EnqueueInstance <IPagingLogger>(_logger.Object); // step1
            hc.EnqueueInstance <IPagingLogger>(_logger.Object); // step2
            hc.EnqueueInstance <IPagingLogger>(_logger.Object); // step3
            hc.EnqueueInstance <IPagingLogger>(_logger.Object); // step4
            hc.EnqueueInstance <IPagingLogger>(_logger.Object); // step5
            hc.EnqueueInstance <IPagingLogger>(_logger.Object); // prepare1
            hc.EnqueueInstance <IPagingLogger>(_logger.Object); // prepare2

            hc.EnqueueInstance <IActionRunner>(step1);
            hc.EnqueueInstance <IActionRunner>(step2);
            hc.EnqueueInstance <IActionRunner>(step3);
            hc.EnqueueInstance <IActionRunner>(step4);
            hc.EnqueueInstance <IActionRunner>(step5);

            _jobEc.Initialize(hc);
            _jobEc.InitializeJob(_message, _tokenSource.Token);
            return(hc);
        }
예제 #14
0
        // Download all required actions.
        // Make sure all condition inputs are valid.
        // Build up three list of steps for jobrunner (pre-job, job, post-job).
        public async Task <List <IStep> > InitializeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message)
        {
            Trace.Entering();
            ArgUtil.NotNull(jobContext, nameof(jobContext));
            ArgUtil.NotNull(message, nameof(message));

            // Create a new timeline record for 'Set up job'
            IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), "Set up job", $"{nameof(JobExtension)}_Init", null, null);

            List <IStep> preJobSteps = new List <IStep>();
            List <IStep> jobSteps    = new List <IStep>();

            using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
            {
                try
                {
                    context.Start();
                    context.Debug($"Starting: Set up job");
                    context.Output($"Current runner version: '{BuildConstants.RunnerPackage.Version}'");
                    var repoFullName = context.GetGitHubContext("repository");
                    ArgUtil.NotNull(repoFullName, nameof(repoFullName));
                    context.Debug($"Primary repository: {repoFullName}");

                    // Print proxy setting information for better diagnostic experience
                    if (!string.IsNullOrEmpty(HostContext.WebProxy.HttpProxyAddress))
                    {
                        context.Output($"Runner is running behind proxy server '{HostContext.WebProxy.HttpProxyAddress}' for all HTTP requests.");
                    }
                    if (!string.IsNullOrEmpty(HostContext.WebProxy.HttpsProxyAddress))
                    {
                        context.Output($"Runner is running behind proxy server '{HostContext.WebProxy.HttpsProxyAddress}' for all HTTPS requests.");
                    }

                    // Prepare the workflow directory
                    context.Output("Prepare workflow directory");
                    var            directoryManager = HostContext.GetService <IPipelineDirectoryManager>();
                    TrackingConfig trackingConfig   = directoryManager.PrepareDirectory(
                        context,
                        message.Workspace);

                    // Set the directory variables
                    context.Debug("Update context data");
                    string _workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
                    context.SetRunnerContext("workspace", Path.Combine(_workDirectory, trackingConfig.PipelineDirectory));
                    context.SetGitHubContext("workspace", Path.Combine(_workDirectory, trackingConfig.WorkspaceDirectory));

                    // Evaluate the job-level environment variables
                    context.Debug("Evaluating job-level environment variables");
                    var templateEvaluator = context.ToPipelineTemplateEvaluator();
                    foreach (var token in message.EnvironmentVariables)
                    {
                        var environmentVariables = templateEvaluator.EvaluateStepEnvironment(token, jobContext.ExpressionValues, VarUtil.EnvironmentVariableKeyComparer);
                        foreach (var pair in environmentVariables)
                        {
                            context.EnvironmentVariables[pair.Key] = pair.Value ?? string.Empty;
                            context.SetEnvContext(pair.Key, pair.Value ?? string.Empty);
                        }
                    }

                    // Evaluate the job container
                    context.Debug("Evaluating job container");
                    var container = templateEvaluator.EvaluateJobContainer(message.JobContainer, jobContext.ExpressionValues);
                    if (container != null)
                    {
                        jobContext.Container = new Container.ContainerInfo(HostContext, container);
                    }

                    // Evaluate the job service containers
                    context.Debug("Evaluating job service containers");
                    var serviceContainers = templateEvaluator.EvaluateJobServiceContainers(message.JobServiceContainers, jobContext.ExpressionValues);
                    if (serviceContainers?.Count > 0)
                    {
                        foreach (var pair in serviceContainers)
                        {
                            var networkAlias     = pair.Key;
                            var serviceContainer = pair.Value;
                            jobContext.ServiceContainers.Add(new Container.ContainerInfo(HostContext, serviceContainer, false, networkAlias));
                        }
                    }

                    // Build up 2 lists of steps, pre-job, job
                    // Download actions not already in the cache
                    Trace.Info("Downloading actions");
                    var actionManager = HostContext.GetService <IActionManager>();
                    var prepareSteps  = await actionManager.PrepareActionsAsync(context, message.Steps);

                    preJobSteps.AddRange(prepareSteps);

                    // Add start-container steps, record and stop-container steps
                    if (jobContext.Container != null || jobContext.ServiceContainers.Count > 0)
                    {
                        var containerProvider = HostContext.GetService <IContainerOperationProvider>();
                        var containers        = new List <Container.ContainerInfo>();
                        if (jobContext.Container != null)
                        {
                            containers.Add(jobContext.Container);
                        }
                        containers.AddRange(jobContext.ServiceContainers);

                        preJobSteps.Add(new JobExtensionRunner(runAsync: containerProvider.StartContainersAsync,
                                                               condition: $"{PipelineTemplateConstants.Success}()",
                                                               displayName: "Initialize containers",
                                                               data: (object)containers));
                    }

                    // Add action steps
                    foreach (var step in message.Steps)
                    {
                        if (step.Type == Pipelines.StepType.Action)
                        {
                            var action = step as Pipelines.ActionStep;
                            Trace.Info($"Adding {action.DisplayName}.");
                            var actionRunner = HostContext.CreateService <IActionRunner>();
                            actionRunner.Action    = action;
                            actionRunner.Stage     = ActionRunStage.Main;
                            actionRunner.Condition = step.Condition;
                            var contextData = new Pipelines.ContextData.DictionaryContextData();
                            if (message.ContextData?.Count > 0)
                            {
                                foreach (var pair in message.ContextData)
                                {
                                    contextData[pair.Key] = pair.Value;
                                }
                            }

                            actionRunner.TryEvaluateDisplayName(contextData, context);
                            jobSteps.Add(actionRunner);
                        }
                    }

                    // Create execution context for pre-job steps
                    foreach (var step in preJobSteps)
                    {
                        if (step is JobExtensionRunner)
                        {
                            JobExtensionRunner extensionStep = step as JobExtensionRunner;
                            ArgUtil.NotNull(extensionStep, extensionStep.DisplayName);
                            Guid stepId = Guid.NewGuid();
                            extensionStep.ExecutionContext = jobContext.CreateChild(stepId, extensionStep.DisplayName, null, null, stepId.ToString("N"));
                        }
                    }

                    // Create execution context for job steps
                    foreach (var step in jobSteps)
                    {
                        if (step is IActionRunner actionStep)
                        {
                            ArgUtil.NotNull(actionStep, step.DisplayName);
                            actionStep.ExecutionContext = jobContext.CreateChild(actionStep.Action.Id, actionStep.DisplayName, actionStep.Action.Name, actionStep.Action.ScopeName, actionStep.Action.ContextName);
                        }
                    }

                    List <IStep> steps = new List <IStep>();
                    steps.AddRange(preJobSteps);
                    steps.AddRange(jobSteps);

                    // Prepare for orphan process cleanup
                    _processCleanup = jobContext.Variables.GetBoolean("process.clean") ?? true;
                    if (_processCleanup)
                    {
                        // Set the RUNNER_TRACKING_ID env variable.
                        Environment.SetEnvironmentVariable(Constants.ProcessTrackingId, _processLookupId);
                        context.Debug("Collect running processes for tracking orphan processes.");

                        // Take a snapshot of current running processes
                        Dictionary <int, Process> processes = SnapshotProcesses();
                        foreach (var proc in processes)
                        {
                            // Pid_ProcessName
                            _existingProcesses.Add($"{proc.Key}_{proc.Value.ProcessName}");
                        }
                    }

                    return(steps);
                }
                catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
                {
                    // Log the exception and cancel the JobExtension Initialization.
                    Trace.Error($"Caught cancellation exception from JobExtension Initialization: {ex}");
                    context.Error(ex);
                    context.Result = TaskResult.Canceled;
                    throw;
                }
                catch (Exception ex)
                {
                    // Log the error and fail the JobExtension Initialization.
                    Trace.Error($"Caught exception from JobExtension Initialization: {ex}");
                    context.Error(ex);
                    context.Result = TaskResult.Failed;
                    throw;
                }
                finally
                {
                    context.Debug("Finishing: Set up job");
                    context.Complete();
                }
            }
        }
예제 #15
0
        public void FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, DateTime jobStartTimeUtc)
        {
            Trace.Entering();
            ArgUtil.NotNull(jobContext, nameof(jobContext));

            // create a new timeline record node for 'Finalize job'
            IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), "Complete job", $"{nameof(JobExtension)}_Final", null, null);

            using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
            {
                try
                {
                    context.Start();
                    context.Debug("Starting: Complete job");

                    if (context.Variables.GetBoolean(Constants.Variables.Actions.RunnerDebug) ?? false)
                    {
                        Trace.Info("Support log upload starting.");
                        context.Output("Uploading runner diagnostic logs");

                        IDiagnosticLogManager diagnosticLogManager = HostContext.GetService <IDiagnosticLogManager>();

                        try
                        {
                            diagnosticLogManager.UploadDiagnosticLogs(executionContext: context, parentContext: jobContext, message: message, jobStartTimeUtc: jobStartTimeUtc);

                            Trace.Info("Support log upload complete.");
                            context.Output("Completed runner diagnostic log upload");
                        }
                        catch (Exception ex)
                        {
                            // Log the error but make sure we continue gracefully.
                            Trace.Info("Error uploading support logs.");
                            context.Output("Error uploading runner diagnostic logs");
                            Trace.Error(ex);
                        }
                    }

                    if (_processCleanup)
                    {
                        context.Output("Cleaning up orphan processes");

                        // Only check environment variable for any process that doesn't run before we invoke our process.
                        Dictionary <int, Process> currentProcesses = SnapshotProcesses();
                        foreach (var proc in currentProcesses)
                        {
                            if (proc.Key == Process.GetCurrentProcess().Id)
                            {
                                // skip for current process.
                                continue;
                            }

                            if (_existingProcesses.Contains($"{proc.Key}_{proc.Value.ProcessName}"))
                            {
                                Trace.Verbose($"Skip existing process. PID: {proc.Key} ({proc.Value.ProcessName})");
                            }
                            else
                            {
                                Trace.Info($"Inspecting process environment variables. PID: {proc.Key} ({proc.Value.ProcessName})");

                                string lookupId = null;
                                try
                                {
                                    lookupId = proc.Value.GetEnvironmentVariable(HostContext, Constants.ProcessTrackingId);
                                }
                                catch (Exception ex)
                                {
                                    Trace.Warning($"Ignore exception during read process environment variables: {ex.Message}");
                                    Trace.Verbose(ex.ToString());
                                }

                                if (string.Equals(lookupId, _processLookupId, StringComparison.OrdinalIgnoreCase))
                                {
                                    context.Output($"Terminate orphan process: pid ({proc.Key}) ({proc.Value.ProcessName})");
                                    try
                                    {
                                        proc.Value.Kill();
                                    }
                                    catch (Exception ex)
                                    {
                                        Trace.Error("Catch exception during orphan process cleanup.");
                                        Trace.Error(ex);
                                    }
                                }
                            }
                        }
                    }
                }
                catch (Exception ex)
                {
                    // Log and ignore the error from JobExtension finalization.
                    Trace.Error($"Caught exception from JobExtension finalization: {ex}");
                    context.Output(ex.Message);
                }
                finally
                {
                    context.Debug("Finishing: Complete job");
                    context.Complete();
                }
            }
        }
예제 #16
0
        public void EchoProcessCommandDebugOn()
        {
            using (TestHostContext _hc = new TestHostContext(this))
            {
                // Set up a few things
                // 1. Job request message (with ACTIONS_STEP_DEBUG = true)
                TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
                TimelineReference timeline          = new TimelineReference();
                Guid   jobId      = Guid.NewGuid();
                string jobName    = "some job name";
                var    jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary <string, VariableValue>(), new List <MaskHint>(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List <Pipelines.ActionStep>(), null);
                jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource()
                {
                    Alias   = Pipelines.PipelineConstants.SelfAlias,
                    Id      = "github",
                    Version = "sha1"
                });
                jobRequest.ContextData["github"]           = new Pipelines.ContextData.DictionaryContextData();
                jobRequest.Variables["ACTIONS_STEP_DEBUG"] = "true";

                // Some service dependencies
                var jobServerQueue = new Mock <IJobServerQueue>();
                jobServerQueue.Setup(x => x.QueueTimelineRecordUpdate(It.IsAny <Guid>(), It.IsAny <TimelineRecord>()));

                _hc.SetSingleton(jobServerQueue.Object);

                var extensionManager = new Mock <IExtensionManager>();
                var echoCommand      = new EchoCommandExtension();
                echoCommand.Initialize(_hc);

                extensionManager.Setup(x => x.GetExtensions <IActionCommandExtension>())
                .Returns(new List <IActionCommandExtension>()
                {
                    echoCommand
                });
                _hc.SetSingleton <IExtensionManager>(extensionManager.Object);

                var configurationStore = new Mock <IConfigurationStore>();
                configurationStore.Setup(x => x.GetSettings()).Returns(new RunnerSettings());
                _hc.SetSingleton(configurationStore.Object);

                var pagingLogger = new Mock <IPagingLogger>();
                _hc.EnqueueInstance(pagingLogger.Object);

                ActionCommandManager commandManager = new ActionCommandManager();
                commandManager.Initialize(_hc);

                var _ec = new Runner.Worker.ExecutionContext();
                _ec.Initialize(_hc);

                // Initialize the job (to exercise logic that sets EchoOnActionCommand)
                _ec.InitializeJob(jobRequest, System.Threading.CancellationToken.None);

                _ec.Complete();

                Assert.True(_ec.EchoOnActionCommand);

                Assert.True(commandManager.TryProcessCommand(_ec, "::echo::off"));
                Assert.False(_ec.EchoOnActionCommand);

                Assert.True(commandManager.TryProcessCommand(_ec, "::echo::on"));
                Assert.True(_ec.EchoOnActionCommand);
            }
        }