コード例 #1
0
        // download all required tasks.
        // make sure all task's condition inputs are valid.
        // build up three list of steps for jobrunner. (pre-job, job, post-job)
        public async Task <JobInitializeResult> InitializeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message)
        {
            Trace.Entering();
            ArgUtil.NotNull(jobContext, nameof(jobContext));
            ArgUtil.NotNull(message, nameof(message));

            // create a new timeline record node for 'Initialize job'
            IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("InitializeJob"), nameof(JobExtension));

            JobInitializeResult initResult = new JobInitializeResult();

            using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
            {
                try
                {
                    context.Start();
                    context.Section(StringUtil.Loc("StepStarting", StringUtil.Loc("InitializeJob")));

                    // Set agent version variable.
                    context.Variables.Set(Constants.Variables.Agent.Version, Constants.Agent.Version);
                    context.Output(StringUtil.Loc("AgentVersion", Constants.Agent.Version));

                    // Print proxy setting information for better diagnostic experience
                    var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>();
                    if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress))
                    {
                        context.Output(StringUtil.Loc("AgentRunningBehindProxy", agentWebProxy.ProxyAddress));
                    }

                    // Give job extension a chance to initialize
                    Trace.Info($"Run initial step from extension {this.GetType().Name}.");
                    InitializeJobExtension(context, message.Steps, message.Workspace);

                    // Download tasks if not already in the cache
                    Trace.Info("Downloading task definitions.");
                    var taskManager = HostContext.GetService <ITaskManager>();
                    await taskManager.DownloadAsync(context, message.Steps);

                    // Parse all Task conditions.
                    Trace.Info("Parsing all task's condition inputs.");
                    var expression = HostContext.GetService <IExpressionManager>();
                    Dictionary <Guid, IExpressionNode> taskConditionMap = new Dictionary <Guid, IExpressionNode>();
                    foreach (var task in message.Steps.OfType <Pipelines.TaskStep>())
                    {
                        IExpressionNode condition;
                        if (!string.IsNullOrEmpty(task.Condition))
                        {
                            context.Debug($"Task '{task.DisplayName}' has following condition: '{task.Condition}'.");
                            condition = expression.Parse(context, task.Condition);
                        }
                        else
                        {
                            condition = ExpressionManager.Succeeded;
                        }

                        taskConditionMap[task.Id] = condition;
                    }

#if OS_WINDOWS
                    // This is for internal testing and is not publicly supported. This will be removed from the agent at a later time.
                    var             prepareScript    = Environment.GetEnvironmentVariable("VSTS_AGENT_INIT_INTERNAL_TEMP_HACK");
                    ServiceEndpoint systemConnection = context.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
                    if (!string.IsNullOrEmpty(prepareScript) && context.Container == null)
                    {
                        var prepareStep = new ManagementScriptStep(
                            scriptPath: prepareScript,
                            condition: ExpressionManager.Succeeded,
                            displayName: "Agent Initialization");

                        Trace.Verbose($"Adding agent init script step.");
                        prepareStep.Initialize(HostContext);
                        prepareStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), prepareStep.DisplayName, nameof(ManagementScriptStep));
                        prepareStep.AccessToken      = systemConnection.Authorization.Parameters["AccessToken"];
                        prepareStep.Condition        = ExpressionManager.Succeeded;
                        initResult.PreJobSteps.Add(prepareStep);
                    }
#endif

                    // build up 3 lists of steps, pre-job, job, post-job
                    Stack <IStep> postJobStepsBuilder = new Stack <IStep>();
                    Dictionary <Guid, Variables> taskVariablesMapping = new Dictionary <Guid, Variables>();

                    if (context.Container != null || context.SidecarContainers.Count > 0)
                    {
                        var containerProvider = HostContext.GetService <IContainerOperationProvider>();
                        var containers        = new List <Container.ContainerInfo>();
                        if (context.Container != null)
                        {
                            containers.Add(context.Container);
                        }
                        containers.AddRange(context.SidecarContainers);

                        initResult.PreJobSteps.Add(new JobExtensionRunner(runAsync: containerProvider.StartContainersAsync,
                                                                          condition: ExpressionManager.Succeeded,
                                                                          displayName: StringUtil.Loc("InitializeContainer"),
                                                                          data: (object)containers));
                        postJobStepsBuilder.Push(new JobExtensionRunner(runAsync: containerProvider.StopContainersAsync,
                                                                        condition: ExpressionManager.Always,
                                                                        displayName: StringUtil.Loc("StopContainer"),
                                                                        data: (object)containers));
                    }

                    foreach (var task in message.Steps.OfType <Pipelines.TaskStep>())
                    {
                        var taskDefinition = taskManager.Load(task);

                        List <string> warnings;
                        taskVariablesMapping[task.Id] = new Variables(HostContext, new Dictionary <string, VariableValue>(), out warnings);

                        // Add pre-job steps from Tasks
                        if (taskDefinition.Data?.PreJobExecution != null)
                        {
                            Trace.Info($"Adding Pre-Job {task.DisplayName}.");
                            var taskRunner = HostContext.CreateService <ITaskRunner>();
                            taskRunner.Task      = task;
                            taskRunner.Stage     = JobRunStage.PreJob;
                            taskRunner.Condition = taskConditionMap[task.Id];
                            initResult.PreJobSteps.Add(taskRunner);
                        }

                        // Add execution steps from Tasks
                        if (taskDefinition.Data?.Execution != null)
                        {
                            Trace.Verbose($"Adding {task.DisplayName}.");
                            var taskRunner = HostContext.CreateService <ITaskRunner>();
                            taskRunner.Task      = task;
                            taskRunner.Stage     = JobRunStage.Main;
                            taskRunner.Condition = taskConditionMap[task.Id];
                            initResult.JobSteps.Add(taskRunner);
                        }

                        // Add post-job steps from Tasks
                        if (taskDefinition.Data?.PostJobExecution != null)
                        {
                            Trace.Verbose($"Adding Post-Job {task.DisplayName}.");
                            var taskRunner = HostContext.CreateService <ITaskRunner>();
                            taskRunner.Task      = task;
                            taskRunner.Stage     = JobRunStage.PostJob;
                            taskRunner.Condition = ExpressionManager.Always;
                            postJobStepsBuilder.Push(taskRunner);
                        }
                    }

                    // Add pre-job step from Extension
                    Trace.Info("Adding pre-job step from extension.");
                    var extensionPreJobStep = GetExtensionPreJobStep(jobContext);
                    if (extensionPreJobStep != null)
                    {
                        initResult.PreJobSteps.Add(extensionPreJobStep);
                    }

                    // Add post-job step from Extension
                    Trace.Info("Adding post-job step from extension.");
                    var extensionPostJobStep = GetExtensionPostJobStep(jobContext);
                    if (extensionPostJobStep != null)
                    {
                        postJobStepsBuilder.Push(extensionPostJobStep);
                    }

                    // create execution context for all pre-job steps
                    foreach (var step in initResult.PreJobSteps)
                    {
#if OS_WINDOWS
                        if (step is ManagementScriptStep)
                        {
                            continue;
                        }
#endif
                        if (step is JobExtensionRunner)
                        {
                            JobExtensionRunner extensionStep = step as JobExtensionRunner;
                            ArgUtil.NotNull(extensionStep, extensionStep.DisplayName);
                            Guid stepId = Guid.NewGuid();
                            extensionStep.ExecutionContext = jobContext.CreateChild(stepId, extensionStep.DisplayName, stepId.ToString("N"));
                        }
                        else if (step is ITaskRunner)
                        {
                            ITaskRunner taskStep = step as ITaskRunner;
                            ArgUtil.NotNull(taskStep, step.DisplayName);
                            taskStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("PreJob", taskStep.DisplayName), taskStep.Task.Name, taskVariablesMapping[taskStep.Task.Id]);
                        }
                    }

                    // create task execution context for all job steps from task
                    foreach (var step in initResult.JobSteps)
                    {
                        ITaskRunner taskStep = step as ITaskRunner;
                        ArgUtil.NotNull(taskStep, step.DisplayName);
                        taskStep.ExecutionContext = jobContext.CreateChild(taskStep.Task.Id, taskStep.DisplayName, taskStep.Task.Name, taskVariablesMapping[taskStep.Task.Id]);
                    }

                    // Add post-job steps from Tasks
                    Trace.Info("Adding post-job steps from tasks.");
                    while (postJobStepsBuilder.Count > 0)
                    {
                        initResult.PostJobStep.Add(postJobStepsBuilder.Pop());
                    }

                    // create task execution context for all post-job steps from task
                    foreach (var step in initResult.PostJobStep)
                    {
                        if (step is JobExtensionRunner)
                        {
                            JobExtensionRunner extensionStep = step as JobExtensionRunner;
                            ArgUtil.NotNull(extensionStep, extensionStep.DisplayName);
                            Guid stepId = Guid.NewGuid();
                            extensionStep.ExecutionContext = jobContext.CreateChild(stepId, extensionStep.DisplayName, stepId.ToString("N"));
                        }
                        else if (step is ITaskRunner)
                        {
                            ITaskRunner taskStep = step as ITaskRunner;
                            ArgUtil.NotNull(taskStep, step.DisplayName);
                            taskStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("PostJob", taskStep.DisplayName), taskStep.Task.Name, taskVariablesMapping[taskStep.Task.Id]);
                        }
                    }

#if OS_WINDOWS
                    // Add script post steps.
                    // This is for internal testing and is not publicly supported. This will be removed from the agent at a later time.
                    var finallyScript = Environment.GetEnvironmentVariable("VSTS_AGENT_CLEANUP_INTERNAL_TEMP_HACK");
                    if (!string.IsNullOrEmpty(finallyScript) && context.Container == null)
                    {
                        var finallyStep = new ManagementScriptStep(
                            scriptPath: finallyScript,
                            condition: ExpressionManager.Always,
                            displayName: "Agent Cleanup");

                        Trace.Verbose($"Adding agent cleanup script step.");
                        finallyStep.Initialize(HostContext);
                        finallyStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), finallyStep.DisplayName, nameof(ManagementScriptStep));
                        finallyStep.Condition        = ExpressionManager.Always;
                        finallyStep.AccessToken      = systemConnection.Authorization.Parameters["AccessToken"];
                        initResult.PostJobStep.Add(finallyStep);
                    }
#endif
                    return(initResult);
                }
                catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
                {
                    // Log the exception and cancel the JobExtension Initialization.
                    Trace.Error($"Caught cancellation exception from JobExtension Initialization: {ex}");
                    context.Error(ex);
                    context.Result = TaskResult.Canceled;
                    throw;
                }
                catch (Exception ex)
                {
                    // Log the error and fail the JobExtension Initialization.
                    Trace.Error($"Caught exception from JobExtension Initialization: {ex}");
                    context.Error(ex);
                    context.Result = TaskResult.Failed;
                    throw;
                }
                finally
                {
                    context.Section(StringUtil.Loc("StepFinishing", StringUtil.Loc("InitializeJob")));
                    context.Complete();
                }
            }
        }
コード例 #2
0
        // download all required tasks.
        // make sure all task's condition inputs are valid.
        // build up three list of steps for jobrunner. (pre-job, job, post-job)
        public async Task <JobInitializeResult> InitializeJob(IExecutionContext jobContext, AgentJobRequestMessage message)
        {
            Trace.Entering();
            ArgUtil.NotNull(jobContext, nameof(jobContext));
            ArgUtil.NotNull(message, nameof(message));

            // create a new timeline record node for 'Initialize job'
            IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("InitializeJob"));

            JobInitializeResult initResult = new JobInitializeResult();

            using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
            {
                try
                {
                    context.Start();
                    context.Section(StringUtil.Loc("StepStarting", StringUtil.Loc("InitializeJob")));

                    // Give job extension a chance to initalize
                    Trace.Info($"Run initial step from extension {this.GetType().Name}.");
                    InitializeJobExtension(context);

                    // Download tasks if not already in the cache
                    Trace.Info("Downloading task definitions.");
                    var taskManager = HostContext.GetService <ITaskManager>();
                    await taskManager.DownloadAsync(context, message.Tasks);

                    // Parse all Task conditions.
                    Trace.Info("Parsing all task's condition inputs.");
                    var expression = HostContext.GetService <IExpressionManager>();
                    Dictionary <Guid, INode> taskConditionMap = new Dictionary <Guid, INode>();
                    foreach (var task in message.Tasks)
                    {
                        INode condition;
                        if (!string.IsNullOrEmpty(task.Condition))
                        {
                            context.Debug($"Task '{task.DisplayName}' has following condition: '{task.Condition}'.");
                            condition = expression.Parse(context, task.Condition);
                        }
                        else if (task.AlwaysRun)
                        {
                            condition = ExpressionManager.SucceededOrFailed;
                        }
                        else
                        {
                            condition = ExpressionManager.Succeeded;
                        }

                        taskConditionMap[task.InstanceId] = condition;
                    }

#if OS_WINDOWS
                    // This is for internal testing and is not publicly supported. This will be removed from the agent at a later time.
                    var prepareScript = Environment.GetEnvironmentVariable("VSTS_AGENT_INIT_INTERNAL_TEMP_HACK");
                    if (!string.IsNullOrEmpty(prepareScript))
                    {
                        var prepareStep = new ManagementScriptStep(
                            scriptPath: prepareScript,
                            condition: ExpressionManager.Succeeded,
                            displayName: "Agent Initialization");

                        Trace.Verbose($"Adding agent init script step.");
                        prepareStep.Initialize(HostContext);
                        prepareStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), prepareStep.DisplayName);
                        prepareStep.AccessToken      = message.Environment.SystemConnection.Authorization.Parameters["AccessToken"];
                        initResult.PreJobSteps.Add(prepareStep);
                    }
#endif

                    // build up 3 lists of steps, pre-job, job, post-job
                    Stack <IStep> postJobStepsBuilder = new Stack <IStep>();
                    Dictionary <Guid, Variables> taskVariablesMapping = new Dictionary <Guid, Variables>();
                    foreach (var taskInstance in message.Tasks)
                    {
                        var taskDefinition = taskManager.Load(taskInstance);

                        List <string> warnings;
                        taskVariablesMapping[taskInstance.InstanceId] = new Variables(HostContext, new Dictionary <string, string>(), message.Environment.MaskHints, out warnings);

                        // Add pre-job steps from Tasks
                        if (taskDefinition.Data?.PreJobExecution != null)
                        {
                            Trace.Info($"Adding Pre-Job {taskInstance.DisplayName}.");
                            var taskRunner = HostContext.CreateService <ITaskRunner>();
                            taskRunner.TaskInstance = taskInstance;
                            taskRunner.Stage        = JobRunStage.PreJob;
                            taskRunner.Condition    = taskConditionMap[taskInstance.InstanceId];
                            initResult.PreJobSteps.Add(taskRunner);
                        }

                        // Add execution steps from Tasks
                        if (taskDefinition.Data?.Execution != null)
                        {
                            Trace.Verbose($"Adding {taskInstance.DisplayName}.");
                            var taskRunner = HostContext.CreateService <ITaskRunner>();
                            taskRunner.TaskInstance = taskInstance;
                            taskRunner.Stage        = JobRunStage.Main;
                            taskRunner.Condition    = taskConditionMap[taskInstance.InstanceId];
                            initResult.JobSteps.Add(taskRunner);
                        }

                        // Add post-job steps from Tasks
                        if (taskDefinition.Data?.PostJobExecution != null)
                        {
                            Trace.Verbose($"Adding Post-Job {taskInstance.DisplayName}.");
                            var taskRunner = HostContext.CreateService <ITaskRunner>();
                            taskRunner.TaskInstance = taskInstance;
                            taskRunner.Stage        = JobRunStage.PostJob;
                            taskRunner.Condition    = taskConditionMap[taskInstance.InstanceId];
                            postJobStepsBuilder.Push(taskRunner);
                        }
                    }

                    // create task execution context for all pre-job steps from task
                    foreach (var step in initResult.PreJobSteps)
                    {
#if OS_WINDOWS
                        if (step is ManagementScriptStep)
                        {
                            continue;
                        }
#endif
                        ITaskRunner taskStep = step as ITaskRunner;
                        ArgUtil.NotNull(taskStep, step.DisplayName);
                        taskStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("PreJob", taskStep.DisplayName), taskVariablesMapping[taskStep.TaskInstance.InstanceId]);
                    }

                    // Add pre-job step from Extension
                    Trace.Info("Adding pre-job step from extension.");
                    var extensionPreJobStep = GetExtensionPreJobStep(jobContext);
                    if (extensionPreJobStep != null)
                    {
                        initResult.PreJobSteps.Add(extensionPreJobStep);
                    }

                    // create task execution context for all job steps from task
                    foreach (var step in initResult.JobSteps)
                    {
                        ITaskRunner taskStep = step as ITaskRunner;
                        ArgUtil.NotNull(taskStep, step.DisplayName);
                        taskStep.ExecutionContext = jobContext.CreateChild(taskStep.TaskInstance.InstanceId, taskStep.DisplayName, taskVariablesMapping[taskStep.TaskInstance.InstanceId]);
                    }

                    // Add post-job steps from Tasks
                    Trace.Info("Adding post-job steps from tasks.");
                    while (postJobStepsBuilder.Count > 0)
                    {
                        initResult.PostJobStep.Add(postJobStepsBuilder.Pop());
                    }

                    // create task execution context for all post-job steps from task
                    foreach (var step in initResult.PostJobStep)
                    {
                        ITaskRunner taskStep = step as ITaskRunner;
                        ArgUtil.NotNull(taskStep, step.DisplayName);
                        taskStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("PostJob", taskStep.DisplayName), taskVariablesMapping[taskStep.TaskInstance.InstanceId]);
                    }

                    // Add post-job step from Extension
                    Trace.Info("Adding post-job step from extension.");
                    var extensionPostJobStep = GetExtensionPostJobStep(jobContext);
                    if (extensionPostJobStep != null)
                    {
                        initResult.PostJobStep.Add(extensionPostJobStep);
                    }

#if OS_WINDOWS
                    // Add script post steps.
                    // This is for internal testing and is not publicly supported. This will be removed from the agent at a later time.
                    var finallyScript = Environment.GetEnvironmentVariable("VSTS_AGENT_CLEANUP_INTERNAL_TEMP_HACK");
                    if (!string.IsNullOrEmpty(finallyScript))
                    {
                        var finallyStep = new ManagementScriptStep(
                            scriptPath: finallyScript,
                            condition: ExpressionManager.Always,
                            displayName: "Agent Cleanup");

                        Trace.Verbose($"Adding agent cleanup script step.");
                        finallyStep.Initialize(HostContext);
                        finallyStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), finallyStep.DisplayName);
                        finallyStep.AccessToken      = message.Environment.SystemConnection.Authorization.Parameters["AccessToken"];
                        initResult.PostJobStep.Add(finallyStep);
                    }
#endif

                    return(initResult);
                }
                catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
                {
                    // Log the exception and cancel the JobExtension Initialization.
                    Trace.Error($"Caught cancellation exception from JobExtension Initialization: {ex}");
                    context.Error(ex);
                    context.Result = TaskResult.Canceled;
                    throw;
                }
                catch (Exception ex)
                {
                    // Log the error and fail the JobExtension Initialization.
                    Trace.Error($"Caught exception from JobExtension Initialization: {ex}");
                    context.Error(ex);
                    context.Result = TaskResult.Failed;
                    throw;
                }
                finally
                {
                    context.Section(StringUtil.Loc("StepFinishing", StringUtil.Loc("InitializeJob")));
                    context.Complete();
                }
            }
        }
コード例 #3
0
        // download all required tasks.
        // make sure all task's condition inputs are valid.
        // build up three list of steps for jobrunner. (pre-job, job, post-job)
        public async Task <List <IStep> > InitializeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message)
        {
            Trace.Entering();
            ArgUtil.NotNull(jobContext, nameof(jobContext));
            ArgUtil.NotNull(message, nameof(message));

            // create a new timeline record node for 'Initialize job'
            IExecutionContext context = jobContext.CreateChild(Guid.NewGuid(), StringUtil.Loc("InitializeJob"), $"{nameof(JobExtension)}_Init");

            List <IStep> preJobSteps  = new List <IStep>();
            List <IStep> jobSteps     = new List <IStep>();
            List <IStep> postJobSteps = new List <IStep>();

            using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); }))
            {
                try
                {
                    context.Start();
                    context.Section(StringUtil.Loc("StepStarting", StringUtil.Loc("InitializeJob")));

                    // Set agent version variable.
                    context.SetVariable(Constants.Variables.Agent.Version, BuildConstants.AgentPackage.Version);
                    context.Output(StringUtil.Loc("AgentNameLog", context.Variables.Get(Constants.Variables.Agent.Name)));
                    context.Output(StringUtil.Loc("AgentMachineNameLog", context.Variables.Get(Constants.Variables.Agent.MachineName)));
                    context.Output(StringUtil.Loc("AgentVersion", BuildConstants.AgentPackage.Version));

                    // Machine specific setup info
                    OutputSetupInfo(context);

                    string imageVersion = System.Environment.GetEnvironmentVariable(Constants.ImageVersionVariable);
                    if (imageVersion != null)
                    {
                        context.Output(StringUtil.Loc("ImageVersionLog", imageVersion));
                    }

                    context.Output(StringUtil.Loc("UserNameLog", System.Environment.UserName));

                    // Print proxy setting information for better diagnostic experience
                    var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>();
                    if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress))
                    {
                        context.Output(StringUtil.Loc("AgentRunningBehindProxy", agentWebProxy.ProxyAddress));
                    }

                    // Give job extension a chance to initialize
                    Trace.Info($"Run initial step from extension {this.GetType().Name}.");
                    InitializeJobExtension(context, message?.Steps, message?.Workspace);

                    // Download tasks if not already in the cache
                    Trace.Info("Downloading task definitions.");
                    var taskManager = HostContext.GetService <ITaskManager>();
                    await taskManager.DownloadAsync(context, message.Steps);

                    // Parse all Task conditions.
                    Trace.Info("Parsing all task's condition inputs.");
                    var expression = HostContext.GetService <IExpressionManager>();
                    Dictionary <Guid, IExpressionNode> taskConditionMap = new Dictionary <Guid, IExpressionNode>();
                    foreach (var task in message.Steps.OfType <Pipelines.TaskStep>())
                    {
                        IExpressionNode condition;
                        if (!string.IsNullOrEmpty(task.Condition))
                        {
                            context.Debug($"Task '{task.DisplayName}' has following condition: '{task.Condition}'.");
                            condition = expression.Parse(context, task.Condition);
                        }
                        else
                        {
                            condition = ExpressionManager.Succeeded;
                        }

                        task.DisplayName = context.Variables.ExpandValue(nameof(task.DisplayName), task.DisplayName);

                        taskConditionMap[task.Id] = condition;
                    }
                    context.Output("Checking job knob settings.");
                    foreach (var knob in Knob.GetAllKnobsFor <AgentKnobs>())
                    {
                        var value = knob.GetValue(jobContext);
                        if (value.Source.GetType() != typeof(BuiltInDefaultKnobSource))
                        {
                            var tag = "";
                            if (knob.IsDeprecated)
                            {
                                tag = "(DEPRECATED)";
                            }
                            else if (knob.IsExperimental)
                            {
                                tag = "(EXPERIMENTAL)";
                            }
                            var outputLine = $"   Knob: {knob.Name} = {value.AsString()} Source: {value.Source.GetDisplayString()} {tag}";
                            if (knob.IsDeprecated)
                            {
                                context.Warning(outputLine);
                            }
                            else
                            {
                                context.Output(outputLine);
                            }
                        }
                    }
                    context.Output("Finished checking job knob settings.");

                    if (PlatformUtil.RunningOnWindows)
                    {
                        // This is for internal testing and is not publicly supported. This will be removed from the agent at a later time.
                        var             prepareScript    = Environment.GetEnvironmentVariable("VSTS_AGENT_INIT_INTERNAL_TEMP_HACK");
                        ServiceEndpoint systemConnection = context.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
                        if (!string.IsNullOrEmpty(prepareScript) && context.StepTarget() is HostInfo)
                        {
                            var prepareStep = new ManagementScriptStep(
                                scriptPath: prepareScript,
                                condition: ExpressionManager.Succeeded,
                                displayName: "Agent Initialization");

                            Trace.Verbose($"Adding agent init script step.");
                            prepareStep.Initialize(HostContext);
                            prepareStep.ExecutionContext = jobContext?.CreateChild(Guid.NewGuid(), prepareStep.DisplayName, nameof(ManagementScriptStep));
                            prepareStep.AccessToken      = systemConnection.Authorization.Parameters["AccessToken"];
                            prepareStep.Condition        = ExpressionManager.Succeeded;
                            preJobSteps.Add(prepareStep);
                        }
                    }

                    // build up 3 lists of steps, pre-job, job, post-job
                    Stack <IStep> postJobStepsBuilder = new Stack <IStep>();
                    Dictionary <Guid, Variables> taskVariablesMapping = new Dictionary <Guid, Variables>();

                    if (context.Containers.Count > 0 || context.SidecarContainers.Count > 0)
                    {
                        var containerProvider = HostContext.GetService <IContainerOperationProvider>();
                        var containers        = new List <ContainerInfo>();
                        containers.AddRange(context.Containers);
                        containers.AddRange(context.SidecarContainers);

                        preJobSteps.Add(new JobExtensionRunner(runAsync: containerProvider.StartContainersAsync,
                                                               condition: ExpressionManager.Succeeded,
                                                               displayName: StringUtil.Loc("InitializeContainer"),
                                                               data: (object)containers));
                        postJobStepsBuilder.Push(new JobExtensionRunner(runAsync: containerProvider.StopContainersAsync,
                                                                        condition: ExpressionManager.Always,
                                                                        displayName: StringUtil.Loc("StopContainer"),
                                                                        data: (object)containers));
                    }

                    Dictionary <Guid, List <TaskRestrictions> > taskRestrictionsMap = new Dictionary <Guid, List <TaskRestrictions> >();
                    foreach (var task in message?.Steps.OfType <Pipelines.TaskStep>())
                    {
                        var taskDefinition = taskManager.Load(task);

                        // Collect any task restrictions from the definition or step
                        var restrictions = new List <TaskRestrictions>();
                        if (taskDefinition.Data.Restrictions != null)
                        {
                            restrictions.Add(taskDefinition.Data.Restrictions);
                        }
                        if (string.Equals(WellKnownStepTargetStrings.Restricted, task.Target?.Commands, StringComparison.OrdinalIgnoreCase))
                        {
                            restrictions.Add(new TaskRestrictions()
                            {
                                Commands = new TaskCommandRestrictions()
                                {
                                    Mode = TaskCommandMode.Restricted
                                }
                            });
                        }
                        if (task.Target?.SettableVariables != null)
                        {
                            restrictions.Add(new TaskRestrictions()
                            {
                                SettableVariables = task.Target.SettableVariables
                            });
                        }
                        taskRestrictionsMap[task.Id] = restrictions;

                        List <string> warnings;
                        taskVariablesMapping[task.Id] = new Variables(HostContext, new Dictionary <string, VariableValue>(), out warnings);

                        // Add pre-job steps from Tasks
                        if (taskDefinition.Data?.PreJobExecution != null)
                        {
                            Trace.Info($"Adding Pre-Job {task.DisplayName}.");
                            var taskRunner = HostContext.CreateService <ITaskRunner>();
                            taskRunner.Task      = task;
                            taskRunner.Stage     = JobRunStage.PreJob;
                            taskRunner.Condition = taskConditionMap[task.Id];
                            preJobSteps.Add(taskRunner);
                        }

                        // Add execution steps from Tasks
                        if (taskDefinition.Data?.Execution != null)
                        {
                            Trace.Verbose($"Adding {task.DisplayName}.");
                            var taskRunner = HostContext.CreateService <ITaskRunner>();
                            taskRunner.Task      = task;
                            taskRunner.Stage     = JobRunStage.Main;
                            taskRunner.Condition = taskConditionMap[task.Id];
                            jobSteps.Add(taskRunner);
                        }

                        // Add post-job steps from Tasks
                        if (taskDefinition.Data?.PostJobExecution != null)
                        {
                            Trace.Verbose($"Adding Post-Job {task.DisplayName}.");
                            var taskRunner = HostContext.CreateService <ITaskRunner>();
                            taskRunner.Task      = task;
                            taskRunner.Stage     = JobRunStage.PostJob;
                            taskRunner.Condition = ExpressionManager.Always;
                            postJobStepsBuilder.Push(taskRunner);
                        }
                    }

                    // Add pre-job step from Extension
                    Trace.Info("Adding pre-job step from extension.");
                    var extensionPreJobStep = GetExtensionPreJobStep(jobContext);
                    if (extensionPreJobStep != null)
                    {
                        preJobSteps.Add(extensionPreJobStep);
                    }

                    // Add post-job step from Extension
                    Trace.Info("Adding post-job step from extension.");
                    var extensionPostJobStep = GetExtensionPostJobStep(jobContext);
                    if (extensionPostJobStep != null)
                    {
                        postJobStepsBuilder.Push(extensionPostJobStep);
                    }

                    ArgUtil.NotNull(jobContext, nameof(jobContext)); // I am not sure why this is needed, but static analysis flagged all uses of jobContext below this point
                    // create execution context for all pre-job steps
                    foreach (var step in preJobSteps)
                    {
                        if (PlatformUtil.RunningOnWindows && step is ManagementScriptStep)
                        {
                            continue;
                        }

                        if (step is JobExtensionRunner)
                        {
                            JobExtensionRunner extensionStep = step as JobExtensionRunner;
                            ArgUtil.NotNull(extensionStep, extensionStep.DisplayName);
                            Guid stepId = Guid.NewGuid();
                            extensionStep.ExecutionContext = jobContext.CreateChild(stepId, extensionStep.DisplayName, stepId.ToString("N"));
                        }
                        else if (step is ITaskRunner)
                        {
                            ITaskRunner taskStep = step as ITaskRunner;
                            ArgUtil.NotNull(taskStep, step.DisplayName);
                            taskStep.ExecutionContext = jobContext.CreateChild(
                                Guid.NewGuid(),
                                StringUtil.Loc("PreJob", taskStep.DisplayName),
                                taskStep.Task.Name,
                                taskVariablesMapping[taskStep.Task.Id],
                                outputForward: true,
                                taskRestrictions: taskRestrictionsMap[taskStep.Task.Id]);
                        }
                    }

                    // create task execution context for all job steps from task
                    foreach (var step in jobSteps)
                    {
                        ITaskRunner taskStep = step as ITaskRunner;
                        ArgUtil.NotNull(taskStep, step.DisplayName);
                        taskStep.ExecutionContext = jobContext.CreateChild(
                            taskStep.Task.Id,
                            taskStep.DisplayName,
                            taskStep.Task.Name,
                            taskVariablesMapping[taskStep.Task.Id],
                            outputForward: true,
                            taskRestrictions: taskRestrictionsMap[taskStep.Task.Id]);
                    }

                    // Add post-job steps from Tasks
                    Trace.Info("Adding post-job steps from tasks.");
                    while (postJobStepsBuilder.Count > 0)
                    {
                        postJobSteps.Add(postJobStepsBuilder.Pop());
                    }

                    // create task execution context for all post-job steps from task
                    foreach (var step in postJobSteps)
                    {
                        if (step is JobExtensionRunner)
                        {
                            JobExtensionRunner extensionStep = step as JobExtensionRunner;
                            ArgUtil.NotNull(extensionStep, extensionStep.DisplayName);
                            Guid stepId = Guid.NewGuid();
                            extensionStep.ExecutionContext = jobContext.CreateChild(stepId, extensionStep.DisplayName, stepId.ToString("N"));
                        }
                        else if (step is ITaskRunner)
                        {
                            ITaskRunner taskStep = step as ITaskRunner;
                            ArgUtil.NotNull(taskStep, step.DisplayName);
                            taskStep.ExecutionContext = jobContext.CreateChild(
                                Guid.NewGuid(),
                                StringUtil.Loc("PostJob", taskStep.DisplayName),
                                taskStep.Task.Name,
                                taskVariablesMapping[taskStep.Task.Id],
                                outputForward: true,
                                taskRestrictions: taskRestrictionsMap[taskStep.Task.Id]);
                        }
                    }

                    if (PlatformUtil.RunningOnWindows)
                    {
                        // Add script post steps.
                        // This is for internal testing and is not publicly supported. This will be removed from the agent at a later time.
                        var finallyScript = Environment.GetEnvironmentVariable("VSTS_AGENT_CLEANUP_INTERNAL_TEMP_HACK");
                        if (!string.IsNullOrEmpty(finallyScript) && context.StepTarget() is HostInfo)
                        {
                            var finallyStep = new ManagementScriptStep(
                                scriptPath: finallyScript,
                                condition: ExpressionManager.Always,
                                displayName: "Agent Cleanup");

                            Trace.Verbose($"Adding agent cleanup script step.");
                            finallyStep.Initialize(HostContext);
                            finallyStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), finallyStep.DisplayName, nameof(ManagementScriptStep));
                            finallyStep.Condition        = ExpressionManager.Always;
                            ServiceEndpoint systemConnection = context.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
                            finallyStep.AccessToken = systemConnection.Authorization.Parameters["AccessToken"];
                            postJobSteps.Add(finallyStep);
                        }
                    }

                    List <IStep> steps = new List <IStep>();
                    steps.AddRange(preJobSteps);
                    steps.AddRange(jobSteps);
                    steps.AddRange(postJobSteps);

                    // Start agent log plugin host process
                    var logPlugin = HostContext.GetService <IAgentLogPlugin>();
                    await logPlugin.StartAsync(context, steps, jobContext.CancellationToken);

                    // Prepare for orphan process cleanup
                    _processCleanup = jobContext.Variables.GetBoolean("process.clean") ?? true;
                    if (_processCleanup)
                    {
                        // Set the VSTS_PROCESS_LOOKUP_ID env variable.
                        context.SetVariable(Constants.ProcessLookupId, _processLookupId, false, false);
                        context.Output("Start tracking orphan processes.");

                        // Take a snapshot of current running processes
                        Dictionary <int, Process> processes = SnapshotProcesses();
                        foreach (var proc in processes)
                        {
                            // Pid_ProcessName
                            _existingProcesses.Add($"{proc.Key}_{proc.Value.ProcessName}");
                        }
                    }
                    _taskKeyCleanup = jobContext.Variables.GetBoolean("process.cleanTaskKey") ?? true;

                    return(steps);
                }
                catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
                {
                    // Log the exception and cancel the JobExtension Initialization.
                    Trace.Error($"Caught cancellation exception from JobExtension Initialization: {ex}");
                    context.Error(ex);
                    context.Result = TaskResult.Canceled;
                    throw;
                }
                catch (Exception ex)
                {
                    // Log the error and fail the JobExtension Initialization.
                    Trace.Error($"Caught exception from JobExtension Initialization: {ex}");
                    context.Error(ex);
                    context.Result = TaskResult.Failed;
                    throw;
                }
                finally
                {
                    context.Section(StringUtil.Loc("StepFinishing", StringUtil.Loc("InitializeJob")));
                    context.Complete();
                }
            }
        }
コード例 #4
0
        public async Task <TaskResult> RunAsync(AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken)
        {
            // Validate parameters.
            Trace.Entering();
            ArgUtil.NotNull(message, nameof(message));
            ArgUtil.NotNull(message.Environment, nameof(message.Environment));
            ArgUtil.NotNull(message.Environment.Variables, nameof(message.Environment.Variables));
            ArgUtil.NotNull(message.Tasks, nameof(message.Tasks));
            Trace.Info("Job ID {0}", message.JobId);

            // System.AccessToken
            if (message.Environment.Variables.ContainsKey(Constants.Variables.System.EnableAccessToken) &&
                StringUtil.ConvertToBoolean(message.Environment.Variables[Constants.Variables.System.EnableAccessToken]))
            {
                // TODO: get access token use Util Method
                message.Environment.Variables[Constants.Variables.System.AccessToken] = message.Environment.SystemConnection.Authorization.Parameters["AccessToken"];
            }

            // Make sure SystemConnection Url and Endpoint Url match Config Url base
            ReplaceConfigUriBaseInJobRequestMessage(message);

            // Setup the job server and job server queue.
            var            jobServer           = HostContext.GetService <IJobServer>();
            VssCredentials jobServerCredential = ApiUtil.GetVssCredential(message.Environment.SystemConnection);
            Uri            jobServerUrl        = message.Environment.SystemConnection.Url;

            Trace.Info($"Creating job server with URL: {jobServerUrl}");
            // jobServerQueue is the throttling reporter.
            _jobServerQueue = HostContext.GetService <IJobServerQueue>();
            VssConnection jobConnection = ApiUtil.CreateConnection(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(_jobServerQueue) });
            await jobServer.ConnectAsync(jobConnection);

            _jobServerQueue.Start(message);

            IExecutionContext jobContext = null;

            try
            {
                // Create the job execution context.
                jobContext = HostContext.CreateService <IExecutionContext>();
                jobContext.InitializeJob(message, jobRequestCancellationToken);
                Trace.Info("Starting the job execution context.");
                jobContext.Start();
                jobContext.Section(StringUtil.Loc("StepStarting", message.JobName));

                // Set agent version into ExecutionContext's variables dictionary.
                jobContext.Variables.Set(Constants.Variables.Agent.Version, Constants.Agent.Version);

                // Print agent version into log for better diagnostic experience
                jobContext.Output(StringUtil.Loc("AgentVersion", Constants.Agent.Version));

                // Print proxy setting information for better diagnostic experience
                var proxyConfig = HostContext.GetService <IProxyConfiguration>();
                if (!string.IsNullOrEmpty(proxyConfig.ProxyUrl))
                {
                    jobContext.Output(StringUtil.Loc("AgentRunningBehindProxy", proxyConfig.ProxyUrl));
                }

                // Validate directory permissions.
                string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);
                Trace.Info($"Validating directory permissions for: '{workDirectory}'");
                try
                {
                    Directory.CreateDirectory(workDirectory);
                    IOUtil.ValidateExecutePermission(workDirectory);
                }
                catch (Exception ex)
                {
                    Trace.Error(ex);
                    jobContext.Error(ex);
                    return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
                }

                // Set agent variables.
                AgentSettings settings = HostContext.GetService <IConfigurationStore>().GetSettings();
                jobContext.Variables.Set(Constants.Variables.Agent.Id, settings.AgentId.ToString(CultureInfo.InvariantCulture));
                jobContext.Variables.Set(Constants.Variables.Agent.HomeDirectory, IOUtil.GetRootPath());
                jobContext.Variables.Set(Constants.Variables.Agent.JobName, message.JobName);
                jobContext.Variables.Set(Constants.Variables.Agent.MachineName, Environment.MachineName);
                jobContext.Variables.Set(Constants.Variables.Agent.Name, settings.AgentName);
                jobContext.Variables.Set(Constants.Variables.Agent.RootDirectory, IOUtil.GetWorkPath(HostContext));
#if OS_WINDOWS
                jobContext.Variables.Set(Constants.Variables.Agent.ServerOMDirectory, Path.Combine(IOUtil.GetExternalsPath(), Constants.Path.ServerOMDirectory));
#endif
                jobContext.Variables.Set(Constants.Variables.Agent.WorkFolder, IOUtil.GetWorkPath(HostContext));
                jobContext.Variables.Set(Constants.Variables.System.WorkFolder, IOUtil.GetWorkPath(HostContext));

                // prefer task definitions url, then TFS collection url, then TFS account url
                var taskServer    = HostContext.GetService <ITaskServer>();
                Uri taskServerUri = null;
                if (!string.IsNullOrEmpty(jobContext.Variables.System_TaskDefinitionsUri))
                {
                    taskServerUri = new Uri(jobContext.Variables.System_TaskDefinitionsUri);
                }
                else if (!string.IsNullOrEmpty(jobContext.Variables.System_TFCollectionUrl))
                {
                    taskServerUri = new Uri(jobContext.Variables.System_TFCollectionUrl);
                }

                var taskServerCredential = ApiUtil.GetVssCredential(message.Environment.SystemConnection);
                if (taskServerUri != null)
                {
                    Trace.Info($"Creating task server with {taskServerUri}");
                    await taskServer.ConnectAsync(ApiUtil.CreateConnection(taskServerUri, taskServerCredential));
                }

                if (taskServerUri == null || !await taskServer.TaskDefinitionEndpointExist(jobRequestCancellationToken))
                {
                    Trace.Info($"Can't determine task download url from JobMessage or the endpoint doesn't exist.");
                    var configStore = HostContext.GetService <IConfigurationStore>();
                    taskServerUri = new Uri(configStore.GetSettings().ServerUrl);
                    Trace.Info($"Recreate task server with configuration server url: {taskServerUri}");
                    await taskServer.ConnectAsync(ApiUtil.CreateConnection(taskServerUri, taskServerCredential));
                }

                // Expand the endpoint data values.
                foreach (ServiceEndpoint endpoint in jobContext.Endpoints)
                {
                    jobContext.Variables.ExpandValues(target: endpoint.Data);
                    VarUtil.ExpandEnvironmentVariables(HostContext, target: endpoint.Data);
                }

                // Get the job extensions.
                Trace.Info("Getting job extensions.");
                string          hostType         = jobContext.Variables.System_HostType;
                var             extensionManager = HostContext.GetService <IExtensionManager>();
                IJobExtension[] extensions       =
                    (extensionManager.GetExtensions <IJobExtension>() ?? new List <IJobExtension>())
                    .Where(x => string.Equals(x.HostType, hostType, StringComparison.OrdinalIgnoreCase))
                    .ToArray();

                List <IStep> steps = new List <IStep>();

#if OS_WINDOWS
                // Init script job extention.
                // This is for internal testing and is not publicly supported. This will be removed from the agent at a later time.
                var prepareScript = Environment.GetEnvironmentVariable("VSTS_AGENT_INIT_INTERNAL_TEMP_HACK");
                if (!string.IsNullOrEmpty(prepareScript))
                {
                    var prepareStep = new ManagementScriptStep(
                        scriptPath: prepareScript,
                        continueOnError: false,
                        critical: true,
                        displayName: "Agent Initialization",
                        enabled: true,
                        @finally: false);

                    Trace.Verbose($"Adding agent init script step.");
                    prepareStep.Initialize(HostContext);
                    prepareStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), prepareStep.DisplayName);
                    prepareStep.AccessToken      = message.Environment.SystemConnection.Authorization.Parameters["AccessToken"];
                    steps.Add(prepareStep);
                }
#endif

                // Add the prepare steps.
                Trace.Info("Adding job prepare extensions.");
                foreach (IJobExtension extension in extensions)
                {
                    if (extension.PrepareStep != null)
                    {
                        Trace.Verbose($"Adding {extension.GetType().Name}.{nameof(extension.PrepareStep)}.");
                        extension.PrepareStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), extension.PrepareStep.DisplayName);
                        steps.Add(extension.PrepareStep);
                    }
                }

                // Add the task steps.
                Trace.Info("Adding tasks.");
                foreach (TaskInstance taskInstance in message.Tasks)
                {
                    Trace.Verbose($"Adding {taskInstance.DisplayName}.");
                    var taskRunner = HostContext.CreateService <ITaskRunner>();
                    taskRunner.ExecutionContext = jobContext.CreateChild(taskInstance.InstanceId, taskInstance.DisplayName);
                    taskRunner.TaskInstance     = taskInstance;
                    steps.Add(taskRunner);
                }

                // Add the finally steps.
                Trace.Info("Adding job finally extensions.");
                foreach (IJobExtension extension in extensions)
                {
                    if (extension.FinallyStep != null)
                    {
                        Trace.Verbose($"Adding {extension.GetType().Name}.{nameof(extension.FinallyStep)}.");
                        extension.FinallyStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), extension.FinallyStep.DisplayName);
                        steps.Add(extension.FinallyStep);
                    }
                }

#if OS_WINDOWS
                // Add script post steps.
                // This is for internal testing and is not publicly supported. This will be removed from the agent at a later time.
                var finallyScript = Environment.GetEnvironmentVariable("VSTS_AGENT_CLEANUP_INTERNAL_TEMP_HACK");
                if (!string.IsNullOrEmpty(finallyScript))
                {
                    var finallyStep = new ManagementScriptStep(
                        scriptPath: finallyScript,
                        continueOnError: false,
                        critical: true,
                        displayName: "Agent Cleanup",
                        enabled: true,
                        @finally: true);

                    Trace.Verbose($"Adding agent cleanup script step.");
                    finallyStep.Initialize(HostContext);
                    finallyStep.ExecutionContext = jobContext.CreateChild(Guid.NewGuid(), finallyStep.DisplayName);
                    finallyStep.AccessToken      = message.Environment.SystemConnection.Authorization.Parameters["AccessToken"];
                    steps.Add(finallyStep);
                }
#endif

                // Download tasks if not already in the cache
                Trace.Info("Downloading task definitions.");
                var taskManager = HostContext.GetService <ITaskManager>();
                try
                {
                    await taskManager.DownloadAsync(jobContext, message.Tasks);
                }
                catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
                {
                    // set the job to canceled
                    // don't log error issue to job ExecutionContext, since server owns the job level issue
                    Trace.Error($"Caught exception: {ex}");
                    return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled));
                }
                catch (Exception ex)
                {
                    // Log the error and fail the job.
                    Trace.Error($"Caught exception from {nameof(TaskManager)}: {ex}");
                    jobContext.Error(ex);
                    return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
                }

                // Run the steps.
                var stepsRunner = HostContext.GetService <IStepsRunner>();
                try
                {
                    await stepsRunner.RunAsync(jobContext, steps);
                }
                catch (OperationCanceledException ex) when(jobContext.CancellationToken.IsCancellationRequested)
                {
                    // set the job to canceled
                    // don't log error issue to job ExecutionContext, since server owns the job level issue
                    Trace.Error($"Caught exception: {ex}");
                    return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled));
                }
                catch (Exception ex)
                {
                    // Log the error and fail the job.
                    Trace.Error($"Caught exception from {nameof(StepsRunner)}: {ex}");
                    jobContext.Error(ex);
                    return(await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed));
                }

                Trace.Info($"Job result: {jobContext.Result}");

                // Complete the job.
                Trace.Info("Completing the job execution context.");
                return(await CompleteJobAsync(jobServer, jobContext, message));
            }
            finally
            {
                await ShutdownQueue();
            }
        }