public ContainerInfo(IHostContext hostContext, Pipelines.ContainerResource container) { this.ContainerName = container.Alias; string containerImage = container.Properties.Get <string>("image"); ArgUtil.NotNullOrEmpty(containerImage, nameof(containerImage)); this.ContainerImage = containerImage; this.ContainerDisplayName = $"{container.Alias}_{Pipelines.Validation.NameValidation.Sanitize(containerImage)}"; this.ContainerRegistryEndpoint = container.Endpoint?.Id ?? Guid.Empty; this.ContainerCreateOptions = container.Properties.Get <string>("options"); this.SkipContainerImagePull = container.Properties.Get <bool>("localimage"); this.ContainerEnvironmentVariables = container.Environment; #if OS_WINDOWS _pathMappings[hostContext.GetDirectory(WellKnownDirectory.Tools)] = "C:\\__t"; // Tool cache folder may come from ENV, so we need a unique folder to avoid collision _pathMappings[hostContext.GetDirectory(WellKnownDirectory.Work)] = "C:\\__w"; _pathMappings[hostContext.GetDirectory(WellKnownDirectory.Root)] = "C:\\__a"; #else _pathMappings[hostContext.GetDirectory(WellKnownDirectory.Tools)] = "/__t"; // Tool cache folder may come from ENV, so we need a unique folder to avoid collision _pathMappings[hostContext.GetDirectory(WellKnownDirectory.Work)] = "/__w"; _pathMappings[hostContext.GetDirectory(WellKnownDirectory.Root)] = "/__a"; #endif }
public ContainerInfo CreateContainerInfo(Pipelines.ContainerResource container, Boolean isJobContainer = true) { ContainerInfo containerInfo = new ContainerInfo(container, isJobContainer); Dictionary <string, string> pathMappings = new Dictionary <string, string>(); if (PlatformUtil.RunningOnWindows) { pathMappings[this.GetDirectory(WellKnownDirectory.Tools)] = "C:\\__t"; // Tool cache folder may come from ENV, so we need a unique folder to avoid collision pathMappings[this.GetDirectory(WellKnownDirectory.Work)] = "C:\\__w"; pathMappings[this.GetDirectory(WellKnownDirectory.Root)] = "C:\\__a"; // add -v '\\.\pipe\docker_engine:\\.\pipe\docker_engine' when they are available (17.09) } else { pathMappings[this.GetDirectory(WellKnownDirectory.Tools)] = "/__t"; // Tool cache folder may come from ENV, so we need a unique folder to avoid collision pathMappings[this.GetDirectory(WellKnownDirectory.Work)] = "/__w"; pathMappings[this.GetDirectory(WellKnownDirectory.Root)] = "/__a"; } if (containerInfo.IsJobContainer && containerInfo.MapDockerSocket) { containerInfo.MountVolumes.Add(new MountVolume("/var/run/docker.sock", "/var/run/docker.sock")); } containerInfo.AddPathMappings(pathMappings); return(containerInfo); }
public ContainerInfo(Pipelines.ContainerResource container, Boolean isJobContainer = true) { this.ContainerName = container.Alias; string containerImage = container.Properties.Get <string>("image"); ArgUtil.NotNullOrEmpty(containerImage, nameof(containerImage)); this.ContainerImage = containerImage; this.ContainerDisplayName = $"{container.Alias}_{Pipelines.Validation.NameValidation.Sanitize(containerImage)}_{Guid.NewGuid().ToString("N").Substring(0, 6)}"; this.ContainerRegistryEndpoint = container.Endpoint?.Id ?? Guid.Empty; this.ContainerCreateOptions = container.Properties.Get <string>("options"); this.SkipContainerImagePull = container.Properties.Get <bool>("localimage"); _environmentVariables = container.Environment != null ? new Dictionary <string, string>(container.Environment) : new Dictionary <string, string>(); this.ContainerCommand = container.Properties.Get <string>("command", defaultValue: ""); this.IsJobContainer = isJobContainer; this._imageOS = PlatformUtil.HostOS; _pathMappings = new Dictionary <string, string>(PlatformUtil.RunningOnWindows ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal); if (container.Ports?.Count > 0) { foreach (var port in container.Ports) { UserPortMappings[port] = port; } } if (container.Volumes?.Count > 0) { foreach (var volume in container.Volumes) { UserMountVolumes[volume] = volume; } } }
public ContainerInfo CreateContainerInfo(Pipelines.ContainerResource container, Boolean isJobContainer = true) { ContainerInfo containerInfo = new ContainerInfo(container, isJobContainer); if (TestUtil.IsWindows()) { // Tool cache folder may come from ENV, so we need a unique folder to avoid collision containerInfo.PathMappings[this.GetDirectory(WellKnownDirectory.Tools)] = "C:\\__t"; containerInfo.PathMappings[this.GetDirectory(WellKnownDirectory.Work)] = "C:\\__w"; containerInfo.PathMappings[this.GetDirectory(WellKnownDirectory.Root)] = "C:\\__a"; // add -v '\\.\pipe\docker_engine:\\.\pipe\docker_engine' when they are available (17.09) } else { // Tool cache folder may come from ENV, so we need a unique folder to avoid collision containerInfo.PathMappings[this.GetDirectory(WellKnownDirectory.Tools)] = "/__t"; containerInfo.PathMappings[this.GetDirectory(WellKnownDirectory.Work)] = "/__w"; containerInfo.PathMappings[this.GetDirectory(WellKnownDirectory.Root)] = "/__a"; if (containerInfo.IsJobContainer) { containerInfo.MountVolumes.Add(new MountVolume("/var/run/docker.sock", "/var/run/docker.sock")); } } return(containerInfo); }
public ContainerInfo(Pipelines.ContainerResource container, Boolean isJobContainer = true) { this.ContainerName = container.Alias; string containerImage = container.Properties.Get <string>("image"); ArgUtil.NotNullOrEmpty(containerImage, nameof(containerImage)); this.ContainerImage = containerImage; this.ContainerDisplayName = $"{container.Alias}_{Pipelines.Validation.NameValidation.Sanitize(containerImage)}_{Guid.NewGuid().ToString("N").Substring(0, 6)}"; this.ContainerRegistryEndpoint = container.Endpoint?.Id ?? Guid.Empty; this.ContainerCreateOptions = container.Properties.Get <string>("options"); this.SkipContainerImagePull = container.Properties.Get <bool>("localimage"); _environmentVariables = container.Environment; this.ContainerCommand = container.Properties.Get <string>("command", defaultValue: ""); this.IsJobContainer = isJobContainer; if (container.Ports?.Count > 0) { foreach (var port in container.Ports) { UserPortMappings[port] = port; } } if (container.Volumes?.Count > 0) { foreach (var volume in container.Volumes) { UserMountVolumes[volume] = volume; } } }
public void TranslateContainerPathForImageOSTestsWindowsToLinux() { var dockerContainer = new Pipelines.ContainerResource() { Alias = "vsts_container_preview", Image = "foo" }; using (TestHostContext hc = CreateTestContext()) { ContainerInfo info = hc.CreateContainerInfo(dockerContainer, isJobContainer: false); info.ImageOS = PlatformUtil.OS.Linux; foreach (var test in new string[][] { new string [] { "C:\\path\\for\\linux", "/path/for/linux" }, new string [] { "c:\\path\\for\\linux", "/path/for/linux" }, new string [] { "D:\\path\\for\\linux", "/path/for/linux" }, new string [] { "C:\\", "/" }, new string [] { "/path/for/linux", "/path/for/linux" }, new string [] { "", "" }, new string [] { null, null }, }) { var winPath = test[0]; var linPath = test[1]; var got = info.TranslateContainerPathForImageOS(PlatformUtil.OS.Windows, winPath); Assert.True(string.Equals(got, linPath), $"Converted {winPath} expected {linPath}, got {got}"); } } }
public void TranslatePathForStepTarget_should_convert_path_only_for_containers(bool isCheckout, string jobTarget, string stepTarget) { // Note: the primary repository is defined as the first repository that is checked out in the job using (TestHostContext hc = CreateTestContext()) using (var ec = new Agent.Worker.ExecutionContext()) { ec.Initialize(hc); // Arrange: Create a container. var pipeContainer = new Pipelines.ContainerResource { Alias = "container" }; pipeContainer.Properties.Set <string>("image", "someimage"); // Arrange: Create a job request message. TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); TimelineReference timeline = new TimelineReference(); JobEnvironment environment = new JobEnvironment(); environment.SystemConnection = new ServiceEndpoint(); List <Pipelines.JobStep> steps = new List <Pipelines.JobStep>(); steps.Add(new Pipelines.TaskStep { Target = new Pipelines.StepTarget { Target = stepTarget }, Reference = new Pipelines.TaskStepDefinitionReference() }); var resources = new Pipelines.JobResources(); resources.Containers.Add(pipeContainer); Guid JobId = Guid.NewGuid(); string jobName = "some job name"; var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, jobTarget, new Dictionary <string, string>(), new Dictionary <string, VariableValue>(), new List <MaskHint>(), resources, new Pipelines.WorkspaceOptions(), steps); // Arrange var pagingLogger = new Mock <IPagingLogger>(); hc.EnqueueInstance(pagingLogger.Object); // Act. ec.InitializeJob(jobRequest, CancellationToken.None); ec.SetStepTarget(steps[0].Target); ec.Variables.Set(Constants.Variables.Task.SkipTranslatorForCheckout, isCheckout.ToString()); string stringBeforeTranslation = hc.GetDirectory(WellKnownDirectory.Work); string stringAfterTranslation = ec.TranslatePathForStepTarget(stringBeforeTranslation); // Assert. if ((stepTarget == "container") || (isCheckout is false && jobTarget == "container" && stepTarget == null)) { string stringContainer = "C:\\__w"; if (ec.StepTarget().ExecutionOS != PlatformUtil.OS.Windows) { stringContainer = "/__w"; } Assert.Equal(stringContainer, stringAfterTranslation); }
public void SidecarContainers_VerifyNotJobContainers() { using (TestHostContext hc = CreateTestContext()) using (var ec = new Agent.Worker.ExecutionContext()) { ec.Initialize(hc); var pipeContainer = new Pipelines.ContainerResource { Alias = "container" }; var pipeContainerSidecar = new Pipelines.ContainerResource { Alias = "sidecar" }; var pipeContainerExtra = new Pipelines.ContainerResource { Alias = "extra" }; pipeContainer.Properties.Set <string>("image", "someimage"); pipeContainerSidecar.Properties.Set <string>("image", "someimage"); pipeContainerExtra.Properties.Set <string>("image", "someimage"); // Arrange: Create a job request message. TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); TimelineReference timeline = new TimelineReference(); JobEnvironment environment = new JobEnvironment(); environment.SystemConnection = new ServiceEndpoint(); List <Pipelines.JobStep> steps = new List <Pipelines.JobStep>(); steps.Add(new Pipelines.TaskStep { Reference = new Pipelines.TaskStepDefinitionReference() }); var resources = new Pipelines.JobResources(); resources.Containers.Add(pipeContainer); resources.Containers.Add(pipeContainerSidecar); resources.Containers.Add(pipeContainerExtra); Guid JobId = Guid.NewGuid(); string jobName = "some job name"; var sidecarContainers = new Dictionary <string, string>(); sidecarContainers.Add("sidecar", "sidecar"); var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, null, sidecarContainers, new Dictionary <string, VariableValue>(), new List <MaskHint>(), resources, new Pipelines.WorkspaceOptions(), steps); // Arrange: Setup command manager var pagingLogger = new Mock <IPagingLogger>(); hc.EnqueueInstance(pagingLogger.Object); // Act. ec.InitializeJob(jobRequest, CancellationToken.None); // Assert. Assert.Equal(2, ec.Containers.Count()); Assert.Equal(1, ec.SidecarContainers.Count()); Assert.False(ec.SidecarContainers.First().IsJobContainer); } }
public ContainerInfo(Pipelines.ContainerResource container) { this.ContainerName = container.Name; string containerImage = container.Properties.Get <string>("image"); ArgUtil.NotNullOrEmpty(containerImage, nameof(containerImage)); this.ContainerImage = containerImage; this.ContainerDisplayName = $"{container.Name}_{Pipelines.Validation.NameValidation.Sanitize(containerImage)}"; this.ContainerRegistryEndpoint = container.Endpoint.Id; this.ContainerCreateOptions = container.Properties.Get <string>("options"); this.SkipContainerImagePull = container.Properties.Get <bool>("localimage"); this.ContainerEnvironmentVariables = container.Environment; }
public void StepTarget_RestrictedCommands_Host() { using (TestHostContext hc = CreateTestContext()) using (var ec = new Agent.Worker.ExecutionContext()) { ec.Initialize(hc); var pipeContainer = new Pipelines.ContainerResource { Alias = "container" }; pipeContainer.Properties.Set <string>("image", "someimage"); // Arrange: Create a job request message. TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); TimelineReference timeline = new TimelineReference(); JobEnvironment environment = new JobEnvironment(); environment.SystemConnection = new ServiceEndpoint(); List <Pipelines.JobStep> steps = new List <Pipelines.JobStep>(); steps.Add(new Pipelines.TaskStep { Target = new Pipelines.StepTarget { Target = "host", Commands = "restricted" }, Reference = new Pipelines.TaskStepDefinitionReference() }); var resources = new Pipelines.JobResources(); resources.Containers.Add(pipeContainer); Guid JobId = Guid.NewGuid(); string jobName = "some job name"; var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, null, new Dictionary <string, string>(), new Dictionary <string, VariableValue>(), new List <MaskHint>(), resources, new Pipelines.WorkspaceOptions(), steps); // Arrange: Setup command manager var commandMock = new Mock <IWorkerCommandManager>(); hc.SetSingleton(commandMock.Object); var pagingLogger = new Mock <IPagingLogger>(); hc.EnqueueInstance(pagingLogger.Object); // Act. ec.InitializeJob(jobRequest, CancellationToken.None); ec.SetStepTarget(steps[0].Target); // Assert. Assert.IsType <HostInfo>(ec.StepTarget()); commandMock.Verify(x => x.SetCommandRestrictionPolicy(It.IsAny <AttributeBasedWorkerCommandRestrictionPolicy>())); } }
public void DefaultContainerInfoMappings() { var dockerContainer = new Pipelines.ContainerResource() { Alias = "vsts_container_preview", Image = "foo" }; using (TestHostContext hc = CreateTestContext()) { ContainerInfo info = hc.CreateContainerInfo(dockerContainer, isJobContainer: false); Assert.True(info.TranslateToContainerPath(hc.GetDirectory(WellKnownDirectory.Tools)).EndsWith($"{Path.DirectorySeparatorChar}__t"), "Tools directory maps"); Assert.True(info.TranslateToContainerPath(hc.GetDirectory(WellKnownDirectory.Work)).EndsWith($"{Path.DirectorySeparatorChar}__w"), "Work directory maps"); Assert.True(info.TranslateToContainerPath(hc.GetDirectory(WellKnownDirectory.Root)).EndsWith($"{Path.DirectorySeparatorChar}__a"), "Root directory maps"); } }
public ContainerInfo(IHostContext hostContext, Pipelines.ContainerResource container, Boolean isJobContainer = true) { this.ContainerName = container.Alias; string containerImage = container.Properties.Get <string>("image"); ArgUtil.NotNullOrEmpty(containerImage, nameof(containerImage)); this.ContainerImage = containerImage; this.ContainerDisplayName = $"{container.Alias}_{Pipelines.Validation.NameValidation.Sanitize(containerImage)}_{Guid.NewGuid().ToString("N").Substring(0, 6)}"; this.ContainerRegistryEndpoint = container.Endpoint?.Id ?? Guid.Empty; this.ContainerCreateOptions = container.Properties.Get <string>("options"); this.SkipContainerImagePull = container.Properties.Get <bool>("localimage"); _environmentVariables = container.Environment; this.ContainerCommand = container.Properties.Get <string>("command", defaultValue: ""); this.IsJobContainer = isJobContainer; #if OS_WINDOWS _pathMappings[hostContext.GetDirectory(WellKnownDirectory.Tools)] = "C:\\__t"; // Tool cache folder may come from ENV, so we need a unique folder to avoid collision _pathMappings[hostContext.GetDirectory(WellKnownDirectory.Work)] = "C:\\__w"; _pathMappings[hostContext.GetDirectory(WellKnownDirectory.Root)] = "C:\\__a"; // add -v '\\.\pipe\docker_engine:\\.\pipe\docker_engine' when they are available (17.09) #else _pathMappings[hostContext.GetDirectory(WellKnownDirectory.Tools)] = "/__t"; // Tool cache folder may come from ENV, so we need a unique folder to avoid collision _pathMappings[hostContext.GetDirectory(WellKnownDirectory.Work)] = "/__w"; _pathMappings[hostContext.GetDirectory(WellKnownDirectory.Root)] = "/__a"; if (this.IsJobContainer) { this.MountVolumes.Add(new MountVolume("/var/run/docker.sock", "/var/run/docker.sock")); } #endif if (container.Ports?.Count > 0) { foreach (var port in container.Ports) { UserPortMappings[port] = port; } } if (container.Volumes?.Count > 0) { foreach (var volume in container.Volumes) { UserMountVolumes[volume] = volume; } } }
public void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token) { // Validation Trace.Entering(); ArgUtil.NotNull(message, nameof(message)); ArgUtil.NotNull(message.Resources, nameof(message.Resources)); ArgUtil.NotNull(message.Variables, nameof(message.Variables)); ArgUtil.NotNull(message.Plan, nameof(message.Plan)); _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token); // Features Features = ApiUtil.GetFeatures(message.Plan); // Endpoints Endpoints = message.Resources.Endpoints; // SecureFiles SecureFiles = message.Resources.SecureFiles; // Variables (constructor performs initial recursive expansion) List <string> warnings; Variables = new Variables(HostContext, message.Variables, out warnings); // Prepend Path PrependPath = new List <string>(); // Docker string imageName = Variables.Get("_PREVIEW_VSTS_DOCKER_IMAGE"); if (string.IsNullOrEmpty(imageName)) { imageName = Environment.GetEnvironmentVariable("_PREVIEW_VSTS_DOCKER_IMAGE"); } if (!string.IsNullOrEmpty(imageName) && string.IsNullOrEmpty(message.JobContainer)) { var dockerContainer = new Pipelines.ContainerResource() { Name = "vsts_container_preview" }; dockerContainer.Properties.Set("image", imageName); Container = new ContainerInfo(dockerContainer); } else if (!string.IsNullOrEmpty(message.JobContainer)) { Container = new ContainerInfo(message.Resources.Containers.Single(x => string.Equals(x.Name, message.JobContainer, StringComparison.OrdinalIgnoreCase))); } else { Container = null; } // Proxy variables var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>(); if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress)) { Variables.Set(Constants.Variables.Agent.ProxyUrl, agentWebProxy.ProxyAddress); Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY", string.Empty); if (!string.IsNullOrEmpty(agentWebProxy.ProxyUsername)) { Variables.Set(Constants.Variables.Agent.ProxyUsername, agentWebProxy.ProxyUsername); Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_USERNAME", string.Empty); } if (!string.IsNullOrEmpty(agentWebProxy.ProxyPassword)) { Variables.Set(Constants.Variables.Agent.ProxyPassword, agentWebProxy.ProxyPassword, true); Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_PASSWORD", string.Empty); } if (agentWebProxy.ProxyBypassList.Count > 0) { Variables.Set(Constants.Variables.Agent.ProxyBypassList, JsonUtility.ToString(agentWebProxy.ProxyBypassList)); } } // Certificate variables var agentCert = HostContext.GetService <IAgentCertificateManager>(); if (agentCert.SkipServerCertificateValidation) { Variables.Set(Constants.Variables.Agent.SslSkipCertValidation, bool.TrueString); } if (!string.IsNullOrEmpty(agentCert.CACertificateFile)) { Variables.Set(Constants.Variables.Agent.SslCAInfo, agentCert.CACertificateFile); } if (!string.IsNullOrEmpty(agentCert.ClientCertificateFile) && !string.IsNullOrEmpty(agentCert.ClientCertificatePrivateKeyFile) && !string.IsNullOrEmpty(agentCert.ClientCertificateArchiveFile)) { Variables.Set(Constants.Variables.Agent.SslClientCert, agentCert.ClientCertificateFile); Variables.Set(Constants.Variables.Agent.SslClientCertKey, agentCert.ClientCertificatePrivateKeyFile); Variables.Set(Constants.Variables.Agent.SslClientCertArchive, agentCert.ClientCertificateArchiveFile); if (!string.IsNullOrEmpty(agentCert.ClientCertificatePassword)) { Variables.Set(Constants.Variables.Agent.SslClientCertPassword, agentCert.ClientCertificatePassword, true); } } // Job timeline record. InitializeTimelineRecord( timelineId: message.Timeline.Id, timelineRecordId: message.JobId, parentTimelineRecordId: null, recordType: ExecutionContextType.Job, displayName: message.JobDisplayName, refName: message.JobName, order: null); // The job timeline record's order is set by server. // Logger (must be initialized before writing warnings). _logger = HostContext.CreateService <IPagingLogger>(); _logger.Setup(_mainTimelineId, _record.Id); // Log warnings from recursive variable expansion. warnings?.ForEach(x => this.Warning(x)); // Verbosity (from system.debug). WriteDebug = Variables.System_Debug ?? false; // Hook up JobServerQueueThrottling event, we will log warning on server tarpit. _jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived; }
public void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token) { // Validation Trace.Entering(); ArgUtil.NotNull(message, nameof(message)); ArgUtil.NotNull(message.Resources, nameof(message.Resources)); ArgUtil.NotNull(message.Variables, nameof(message.Variables)); ArgUtil.NotNull(message.Plan, nameof(message.Plan)); _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token); // Features Features = PlanUtil.GetFeatures(message.Plan); // Endpoints Endpoints = message.Resources.Endpoints; // SecureFiles SecureFiles = message.Resources.SecureFiles; // Repositories Repositories = message.Resources.Repositories; // JobSettings JobSettings = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase); JobSettings[WellKnownJobSettings.HasMultipleCheckouts] = message.Steps?.Where(x => Pipelines.PipelineConstants.IsCheckoutTask(x)).Count() > 1 ? Boolean.TrueString : Boolean.FalseString; // Variables (constructor performs initial recursive expansion) List <string> warnings; Variables = new Variables(HostContext, message.Variables, out warnings); // Prepend Path PrependPath = new List <string>(); // Docker (JobContainer) string imageName = Variables.Get("_PREVIEW_VSTS_DOCKER_IMAGE"); if (string.IsNullOrEmpty(imageName)) { imageName = Environment.GetEnvironmentVariable("_PREVIEW_VSTS_DOCKER_IMAGE"); } if (!string.IsNullOrEmpty(imageName) && string.IsNullOrEmpty(message.JobContainer)) { var dockerContainer = new Pipelines.ContainerResource() { Alias = "vsts_container_preview" }; dockerContainer.Properties.Set("image", imageName); Container = HostContext.CreateContainerInfo(dockerContainer); } else if (!string.IsNullOrEmpty(message.JobContainer)) { Container = HostContext.CreateContainerInfo(message.Resources.Containers.Single(x => string.Equals(x.Alias, message.JobContainer, StringComparison.OrdinalIgnoreCase))); } else { Container = null; } if (Container != null) { Container.ImageOSChanged += HandleContainerImageOSChange; } // Docker (Sidecar Containers) SidecarContainers = new List <ContainerInfo>(); foreach (var sidecar in message.JobSidecarContainers) { var networkAlias = sidecar.Key; var containerResourceAlias = sidecar.Value; var containerResource = message.Resources.Containers.Single(c => string.Equals(c.Alias, containerResourceAlias, StringComparison.OrdinalIgnoreCase)); ContainerInfo containerInfo = HostContext.CreateContainerInfo(containerResource, isJobContainer: false); containerInfo.ContainerNetworkAlias = networkAlias; SidecarContainers.Add(containerInfo); } // Proxy variables var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>(); if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress)) { Variables.Set(Constants.Variables.Agent.ProxyUrl, agentWebProxy.ProxyAddress); Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY", string.Empty); if (!string.IsNullOrEmpty(agentWebProxy.ProxyUsername)) { Variables.Set(Constants.Variables.Agent.ProxyUsername, agentWebProxy.ProxyUsername); Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_USERNAME", string.Empty); } if (!string.IsNullOrEmpty(agentWebProxy.ProxyPassword)) { Variables.Set(Constants.Variables.Agent.ProxyPassword, agentWebProxy.ProxyPassword, true); Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_PASSWORD", string.Empty); } if (agentWebProxy.ProxyBypassList.Count > 0) { Variables.Set(Constants.Variables.Agent.ProxyBypassList, JsonUtility.ToString(agentWebProxy.ProxyBypassList)); } } // Certificate variables var agentCert = HostContext.GetService <IAgentCertificateManager>(); if (agentCert.SkipServerCertificateValidation) { Variables.Set(Constants.Variables.Agent.SslSkipCertValidation, bool.TrueString); } if (!string.IsNullOrEmpty(agentCert.CACertificateFile)) { Variables.Set(Constants.Variables.Agent.SslCAInfo, agentCert.CACertificateFile); } if (!string.IsNullOrEmpty(agentCert.ClientCertificateFile) && !string.IsNullOrEmpty(agentCert.ClientCertificatePrivateKeyFile) && !string.IsNullOrEmpty(agentCert.ClientCertificateArchiveFile)) { Variables.Set(Constants.Variables.Agent.SslClientCert, agentCert.ClientCertificateFile); Variables.Set(Constants.Variables.Agent.SslClientCertKey, agentCert.ClientCertificatePrivateKeyFile); Variables.Set(Constants.Variables.Agent.SslClientCertArchive, agentCert.ClientCertificateArchiveFile); if (!string.IsNullOrEmpty(agentCert.ClientCertificatePassword)) { Variables.Set(Constants.Variables.Agent.SslClientCertPassword, agentCert.ClientCertificatePassword, true); } } // Runtime option variables var runtimeOptions = HostContext.GetService <IConfigurationStore>().GetAgentRuntimeOptions(); if (runtimeOptions != null) { if (PlatformUtil.RunningOnWindows && runtimeOptions.GitUseSecureChannel) { Variables.Set(Constants.Variables.Agent.GitUseSChannel, runtimeOptions.GitUseSecureChannel.ToString()); } } // Job timeline record. InitializeTimelineRecord( timelineId: message.Timeline.Id, timelineRecordId: message.JobId, parentTimelineRecordId: null, recordType: ExecutionContextType.Job, displayName: message.JobDisplayName, refName: message.JobName, order: null); // The job timeline record's order is set by server. // Logger (must be initialized before writing warnings). _logger = HostContext.CreateService <IPagingLogger>(); _logger.Setup(_mainTimelineId, _record.Id); // Log warnings from recursive variable expansion. warnings?.ForEach(x => this.Warning(x)); // Verbosity (from system.debug). WriteDebug = Variables.System_Debug ?? false; // Hook up JobServerQueueThrottling event, we will log warning on server tarpit. _jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived; }
public void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token) { // Validation Trace.Entering(); ArgUtil.NotNull(message, nameof(message)); ArgUtil.NotNull(message.Resources, nameof(message.Resources)); ArgUtil.NotNull(message.Variables, nameof(message.Variables)); ArgUtil.NotNull(message.Plan, nameof(message.Plan)); _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token); // Features Features = PlanUtil.GetFeatures(message.Plan); // Endpoints Endpoints = message.Resources.Endpoints; // SecureFiles SecureFiles = message.Resources.SecureFiles; // Repositories Repositories = message.Resources.Repositories; // JobSettings var checkouts = message.Steps?.Where(x => Pipelines.PipelineConstants.IsCheckoutTask(x)).ToList(); JobSettings = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase); JobSettings[WellKnownJobSettings.HasMultipleCheckouts] = Boolean.FalseString; if (checkouts != null && checkouts.Count > 0) { JobSettings[WellKnownJobSettings.HasMultipleCheckouts] = checkouts.Count > 1 ? Boolean.TrueString : Boolean.FalseString; var firstCheckout = checkouts.First() as Pipelines.TaskStep; if (firstCheckout != null && Repositories != null && firstCheckout.Inputs.TryGetValue(Pipelines.PipelineConstants.CheckoutTaskInputs.Repository, out string repoAlias)) { JobSettings[WellKnownJobSettings.FirstRepositoryCheckedOut] = repoAlias; var repo = Repositories.Find(r => String.Equals(r.Alias, repoAlias, StringComparison.OrdinalIgnoreCase)); if (repo != null) { repo.Properties.Set <bool>(RepositoryUtil.IsPrimaryRepository, true); } } } // Variables (constructor performs initial recursive expansion) List <string> warnings; Variables = new Variables(HostContext, message.Variables, out warnings); Variables.StringTranslator = TranslatePathForStepTarget; if (Variables.GetBoolean("agent.useWorkspaceId") == true) { try { // We need an identifier that represents which repos make up the workspace. // This allows similar jobs in the same definition to reuse that workspace and other jobs to have their own. JobSettings[WellKnownJobSettings.WorkspaceIdentifier] = GetWorkspaceIdentifier(message); } catch (Exception ex) { Trace.Warning($"Unable to generate workspace ID: {ex.Message}"); } } // Prepend Path PrependPath = new List <string>(); // Docker (JobContainer) string imageName = Variables.Get("_PREVIEW_VSTS_DOCKER_IMAGE"); if (string.IsNullOrEmpty(imageName)) { imageName = Environment.GetEnvironmentVariable("_PREVIEW_VSTS_DOCKER_IMAGE"); } Containers = new List <ContainerInfo>(); _defaultStepTarget = null; _currentStepTarget = null; if (!string.IsNullOrEmpty(imageName) && string.IsNullOrEmpty(message.JobContainer)) { var dockerContainer = new Pipelines.ContainerResource() { Alias = "vsts_container_preview" }; dockerContainer.Properties.Set("image", imageName); var defaultJobContainer = HostContext.CreateContainerInfo(dockerContainer); _defaultStepTarget = defaultJobContainer; Containers.Add(defaultJobContainer); } else if (!string.IsNullOrEmpty(message.JobContainer)) { var defaultJobContainer = HostContext.CreateContainerInfo(message.Resources.Containers.Single(x => string.Equals(x.Alias, message.JobContainer, StringComparison.OrdinalIgnoreCase))); _defaultStepTarget = defaultJobContainer; Containers.Add(defaultJobContainer); } else { _defaultStepTarget = new HostInfo(); } // Include other step containers var sidecarContainers = new HashSet <string>(message.JobSidecarContainers.Values, StringComparer.OrdinalIgnoreCase); foreach (var container in message.Resources.Containers.Where(x => !string.Equals(x.Alias, message.JobContainer, StringComparison.OrdinalIgnoreCase) && !sidecarContainers.Contains(x.Alias))) { Containers.Add(HostContext.CreateContainerInfo(container)); } // Docker (Sidecar Containers) SidecarContainers = new List <ContainerInfo>(); foreach (var sidecar in message.JobSidecarContainers) { var networkAlias = sidecar.Key; var containerResourceAlias = sidecar.Value; var containerResource = message.Resources.Containers.Single(c => string.Equals(c.Alias, containerResourceAlias, StringComparison.OrdinalIgnoreCase)); ContainerInfo containerInfo = HostContext.CreateContainerInfo(containerResource, isJobContainer: false); containerInfo.ContainerNetworkAlias = networkAlias; SidecarContainers.Add(containerInfo); } // Proxy variables var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>(); if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress)) { Variables.Set(Constants.Variables.Agent.ProxyUrl, agentWebProxy.ProxyAddress); Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY", string.Empty); if (!string.IsNullOrEmpty(agentWebProxy.ProxyUsername)) { Variables.Set(Constants.Variables.Agent.ProxyUsername, agentWebProxy.ProxyUsername); Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_USERNAME", string.Empty); } if (!string.IsNullOrEmpty(agentWebProxy.ProxyPassword)) { Variables.Set(Constants.Variables.Agent.ProxyPassword, agentWebProxy.ProxyPassword, true); Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_PASSWORD", string.Empty); } if (agentWebProxy.ProxyBypassList.Count > 0) { Variables.Set(Constants.Variables.Agent.ProxyBypassList, JsonUtility.ToString(agentWebProxy.ProxyBypassList)); } } // Certificate variables var agentCert = HostContext.GetService <IAgentCertificateManager>(); if (agentCert.SkipServerCertificateValidation) { Variables.Set(Constants.Variables.Agent.SslSkipCertValidation, bool.TrueString); } if (!string.IsNullOrEmpty(agentCert.CACertificateFile)) { Variables.Set(Constants.Variables.Agent.SslCAInfo, agentCert.CACertificateFile); } if (!string.IsNullOrEmpty(agentCert.ClientCertificateFile) && !string.IsNullOrEmpty(agentCert.ClientCertificatePrivateKeyFile) && !string.IsNullOrEmpty(agentCert.ClientCertificateArchiveFile)) { Variables.Set(Constants.Variables.Agent.SslClientCert, agentCert.ClientCertificateFile); Variables.Set(Constants.Variables.Agent.SslClientCertKey, agentCert.ClientCertificatePrivateKeyFile); Variables.Set(Constants.Variables.Agent.SslClientCertArchive, agentCert.ClientCertificateArchiveFile); if (!string.IsNullOrEmpty(agentCert.ClientCertificatePassword)) { Variables.Set(Constants.Variables.Agent.SslClientCertPassword, agentCert.ClientCertificatePassword, true); } } // Runtime option variables var runtimeOptions = HostContext.GetService <IConfigurationStore>().GetAgentRuntimeOptions(); if (runtimeOptions != null) { if (PlatformUtil.RunningOnWindows && runtimeOptions.GitUseSecureChannel) { Variables.Set(Constants.Variables.Agent.GitUseSChannel, runtimeOptions.GitUseSecureChannel.ToString()); } } // Job timeline record. InitializeTimelineRecord( timelineId: message.Timeline.Id, timelineRecordId: message.JobId, parentTimelineRecordId: null, recordType: ExecutionContextType.Job, displayName: message.JobDisplayName, refName: message.JobName, order: null); // The job timeline record's order is set by server. // Logger (must be initialized before writing warnings). _logger = HostContext.CreateService <IPagingLogger>(); _logger.Setup(_mainTimelineId, _record.Id); // Log warnings from recursive variable expansion. warnings?.ForEach(x => this.Warning(x)); // Verbosity (from system.debug). WriteDebug = Variables.System_Debug ?? false; // Hook up JobServerQueueThrottling event, we will log warning on server tarpit. _jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived; }
public void GeneratePluginExecutionContextContainerInfoTest() { var dockerContainer = new Pipelines.ContainerResource() { Alias = "vsts_container_preview", Image = "foo" }; using (TestHostContext tc = CreateTestContext()) { Tracing trace = tc.GetTrace(); var agentPluginManager = new AgentPluginManager(); agentPluginManager.Initialize(tc); var containerInfo = tc.CreateContainerInfo(dockerContainer, isJobContainer: false); var containerWorkPath = "/__w"; if (TestUtil.IsWindows()) { containerWorkPath = "C:\\__w"; } var inputs = new Dictionary <string, string>() { { "input1", "foo" }, { "input2", containerWorkPath }, { "input3", tc.GetDirectory(WellKnownDirectory.Work) }, }; var expectedInputs = new Dictionary <string, string>() { { "input1", "foo" }, { "input2", tc.GetDirectory(WellKnownDirectory.Work) }, { "input3", tc.GetDirectory(WellKnownDirectory.Work) }, }; var variables = new Dictionary <string, VariableValue>() { { "variable1", "foo" }, { "variable2", containerWorkPath }, { "variable3", tc.GetDirectory(WellKnownDirectory.Work) }, }; var expectedVariables = new Dictionary <string, VariableValue>() { { "variable1", "foo" }, { "variable2", tc.GetDirectory(WellKnownDirectory.Work) }, { "variable3", tc.GetDirectory(WellKnownDirectory.Work) }, }; var taskVariables = new Dictionary <string, VariableValue>() { { "taskVariable1", "foo" }, { "taskVariable2", containerWorkPath }, { "taskVariable3", tc.GetDirectory(WellKnownDirectory.Work) }, }; var expectedTaskVariables = new Dictionary <string, VariableValue>() { { "taskVariable1", "foo" }, { "taskVariable2", tc.GetDirectory(WellKnownDirectory.Work) }, { "taskVariable3", tc.GetDirectory(WellKnownDirectory.Work) }, }; var executionContext = CreateTestExecutionContext(tc, stepTarget: containerInfo, variables: variables, taskVariables: taskVariables); var pluginContext = agentPluginManager.GeneratePluginExecutionContext(executionContext, inputs, executionContext.Variables); Assert.True(pluginContext != null, "PluginContext for Container Step Target is not null"); Assert.True(expectedInputs.All(e => pluginContext.Inputs.Contains(e))); Assert.True(expectedVariables.All(e => pluginContext.Variables.Contains(e))); Assert.True(expectedTaskVariables.All(e => pluginContext.TaskVariables.Contains(e))); } }