public ContainerInfo(IHostContext hostContext, Pipelines.ContainerResource container)
{
this.ContainerName = container.Alias;
string containerImage = container.Properties.Get <string>("image");
ArgUtil.NotNullOrEmpty(containerImage, nameof(containerImage));
this.ContainerImage = containerImage;
this.ContainerDisplayName = $"{container.Alias}_{Pipelines.Validation.NameValidation.Sanitize(containerImage)}";
this.ContainerRegistryEndpoint = container.Endpoint?.Id ?? Guid.Empty;
this.ContainerCreateOptions = container.Properties.Get <string>("options");
this.SkipContainerImagePull = container.Properties.Get <bool>("localimage");
this.ContainerEnvironmentVariables = container.Environment;
#if OS_WINDOWS
_pathMappings[hostContext.GetDirectory(WellKnownDirectory.Tools)] = "C:\\__t"; // Tool cache folder may come from ENV, so we need a unique folder to avoid collision
_pathMappings[hostContext.GetDirectory(WellKnownDirectory.Work)] = "C:\\__w";
_pathMappings[hostContext.GetDirectory(WellKnownDirectory.Root)] = "C:\\__a";
#else
_pathMappings[hostContext.GetDirectory(WellKnownDirectory.Tools)] = "/__t"; // Tool cache folder may come from ENV, so we need a unique folder to avoid collision
_pathMappings[hostContext.GetDirectory(WellKnownDirectory.Work)] = "/__w";
_pathMappings[hostContext.GetDirectory(WellKnownDirectory.Root)] = "/__a";
#endif
}
public ContainerInfo CreateContainerInfo(Pipelines.ContainerResource container, Boolean isJobContainer = true)
{
ContainerInfo containerInfo = new ContainerInfo(container, isJobContainer);
Dictionary <string, string> pathMappings = new Dictionary <string, string>();
if (PlatformUtil.RunningOnWindows)
{
pathMappings[this.GetDirectory(WellKnownDirectory.Tools)] = "C:\\__t"; // Tool cache folder may come from ENV, so we need a unique folder to avoid collision
pathMappings[this.GetDirectory(WellKnownDirectory.Work)] = "C:\\__w";
pathMappings[this.GetDirectory(WellKnownDirectory.Root)] = "C:\\__a";
// add -v '\\.\pipe\docker_engine:\\.\pipe\docker_engine' when they are available (17.09)
}
else
{
pathMappings[this.GetDirectory(WellKnownDirectory.Tools)] = "/__t"; // Tool cache folder may come from ENV, so we need a unique folder to avoid collision
pathMappings[this.GetDirectory(WellKnownDirectory.Work)] = "/__w";
pathMappings[this.GetDirectory(WellKnownDirectory.Root)] = "/__a";
}
if (containerInfo.IsJobContainer && containerInfo.MapDockerSocket)
{
containerInfo.MountVolumes.Add(new MountVolume("/var/run/docker.sock", "/var/run/docker.sock"));
}
containerInfo.AddPathMappings(pathMappings);
return(containerInfo);
}
public ContainerInfo(Pipelines.ContainerResource container, Boolean isJobContainer = true)
{
this.ContainerName = container.Alias;
string containerImage = container.Properties.Get <string>("image");
ArgUtil.NotNullOrEmpty(containerImage, nameof(containerImage));
this.ContainerImage = containerImage;
this.ContainerDisplayName = $"{container.Alias}_{Pipelines.Validation.NameValidation.Sanitize(containerImage)}_{Guid.NewGuid().ToString("N").Substring(0, 6)}";
this.ContainerRegistryEndpoint = container.Endpoint?.Id ?? Guid.Empty;
this.ContainerCreateOptions = container.Properties.Get <string>("options");
this.SkipContainerImagePull = container.Properties.Get <bool>("localimage");
_environmentVariables = container.Environment != null ? new Dictionary <string, string>(container.Environment) : new Dictionary <string, string>();
this.ContainerCommand = container.Properties.Get <string>("command", defaultValue: "");
this.IsJobContainer = isJobContainer;
this._imageOS = PlatformUtil.HostOS;
_pathMappings = new Dictionary <string, string>(PlatformUtil.RunningOnWindows ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal);
if (container.Ports?.Count > 0)
{
foreach (var port in container.Ports)
{
UserPortMappings[port] = port;
}
}
if (container.Volumes?.Count > 0)
{
foreach (var volume in container.Volumes)
{
UserMountVolumes[volume] = volume;
}
}
}
public ContainerInfo CreateContainerInfo(Pipelines.ContainerResource container, Boolean isJobContainer = true)
{
ContainerInfo containerInfo = new ContainerInfo(container, isJobContainer);
if (TestUtil.IsWindows())
{
// Tool cache folder may come from ENV, so we need a unique folder to avoid collision
containerInfo.PathMappings[this.GetDirectory(WellKnownDirectory.Tools)] = "C:\\__t";
containerInfo.PathMappings[this.GetDirectory(WellKnownDirectory.Work)] = "C:\\__w";
containerInfo.PathMappings[this.GetDirectory(WellKnownDirectory.Root)] = "C:\\__a";
// add -v '\\.\pipe\docker_engine:\\.\pipe\docker_engine' when they are available (17.09)
}
else
{
// Tool cache folder may come from ENV, so we need a unique folder to avoid collision
containerInfo.PathMappings[this.GetDirectory(WellKnownDirectory.Tools)] = "/__t";
containerInfo.PathMappings[this.GetDirectory(WellKnownDirectory.Work)] = "/__w";
containerInfo.PathMappings[this.GetDirectory(WellKnownDirectory.Root)] = "/__a";
if (containerInfo.IsJobContainer)
{
containerInfo.MountVolumes.Add(new MountVolume("/var/run/docker.sock", "/var/run/docker.sock"));
}
}
return(containerInfo);
}
public ContainerInfo(Pipelines.ContainerResource container, Boolean isJobContainer = true)
{
this.ContainerName = container.Alias;
string containerImage = container.Properties.Get <string>("image");
ArgUtil.NotNullOrEmpty(containerImage, nameof(containerImage));
this.ContainerImage = containerImage;
this.ContainerDisplayName = $"{container.Alias}_{Pipelines.Validation.NameValidation.Sanitize(containerImage)}_{Guid.NewGuid().ToString("N").Substring(0, 6)}";
this.ContainerRegistryEndpoint = container.Endpoint?.Id ?? Guid.Empty;
this.ContainerCreateOptions = container.Properties.Get <string>("options");
this.SkipContainerImagePull = container.Properties.Get <bool>("localimage");
_environmentVariables = container.Environment;
this.ContainerCommand = container.Properties.Get <string>("command", defaultValue: "");
this.IsJobContainer = isJobContainer;
if (container.Ports?.Count > 0)
{
foreach (var port in container.Ports)
{
UserPortMappings[port] = port;
}
}
if (container.Volumes?.Count > 0)
{
foreach (var volume in container.Volumes)
{
UserMountVolumes[volume] = volume;
}
}
}
public void TranslateContainerPathForImageOSTestsWindowsToLinux()
{
var dockerContainer = new Pipelines.ContainerResource()
{
Alias = "vsts_container_preview",
Image = "foo"
};
using (TestHostContext hc = CreateTestContext())
{
ContainerInfo info = hc.CreateContainerInfo(dockerContainer, isJobContainer: false);
info.ImageOS = PlatformUtil.OS.Linux;
foreach (var test in new string[][] {
new string [] { "C:\\path\\for\\linux", "/path/for/linux" },
new string [] { "c:\\path\\for\\linux", "/path/for/linux" },
new string [] { "D:\\path\\for\\linux", "/path/for/linux" },
new string [] { "C:\\", "/" },
new string [] { "/path/for/linux", "/path/for/linux" },
new string [] { "", "" },
new string [] { null, null },
})
{
var winPath = test[0];
var linPath = test[1];
var got = info.TranslateContainerPathForImageOS(PlatformUtil.OS.Windows, winPath);
Assert.True(string.Equals(got, linPath), $"Converted {winPath} expected {linPath}, got {got}");
}
}
}
public void TranslatePathForStepTarget_should_convert_path_only_for_containers(bool isCheckout, string jobTarget, string stepTarget)
{
// Note: the primary repository is defined as the first repository that is checked out in the job
using (TestHostContext hc = CreateTestContext())
using (var ec = new Agent.Worker.ExecutionContext())
{
ec.Initialize(hc);
// Arrange: Create a container.
var pipeContainer = new Pipelines.ContainerResource
{
Alias = "container"
};
pipeContainer.Properties.Set <string>("image", "someimage");
// Arrange: Create a job request message.
TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
TimelineReference timeline = new TimelineReference();
JobEnvironment environment = new JobEnvironment();
environment.SystemConnection = new ServiceEndpoint();
List <Pipelines.JobStep> steps = new List <Pipelines.JobStep>();
steps.Add(new Pipelines.TaskStep
{
Target = new Pipelines.StepTarget
{
Target = stepTarget
},
Reference = new Pipelines.TaskStepDefinitionReference()
});
var resources = new Pipelines.JobResources();
resources.Containers.Add(pipeContainer);
Guid JobId = Guid.NewGuid();
string jobName = "some job name";
var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, jobTarget, new Dictionary <string, string>(),
new Dictionary <string, VariableValue>(), new List <MaskHint>(), resources, new Pipelines.WorkspaceOptions(), steps);
// Arrange
var pagingLogger = new Mock <IPagingLogger>();
hc.EnqueueInstance(pagingLogger.Object);
// Act.
ec.InitializeJob(jobRequest, CancellationToken.None);
ec.SetStepTarget(steps[0].Target);
ec.Variables.Set(Constants.Variables.Task.SkipTranslatorForCheckout, isCheckout.ToString());
string stringBeforeTranslation = hc.GetDirectory(WellKnownDirectory.Work);
string stringAfterTranslation = ec.TranslatePathForStepTarget(stringBeforeTranslation);
// Assert.
if ((stepTarget == "container") || (isCheckout is false && jobTarget == "container" && stepTarget == null))
{
string stringContainer = "C:\\__w";
if (ec.StepTarget().ExecutionOS != PlatformUtil.OS.Windows)
{
stringContainer = "/__w";
}
Assert.Equal(stringContainer, stringAfterTranslation);
}
public void SidecarContainers_VerifyNotJobContainers()
{
using (TestHostContext hc = CreateTestContext())
using (var ec = new Agent.Worker.ExecutionContext())
{
ec.Initialize(hc);
var pipeContainer = new Pipelines.ContainerResource {
Alias = "container"
};
var pipeContainerSidecar = new Pipelines.ContainerResource {
Alias = "sidecar"
};
var pipeContainerExtra = new Pipelines.ContainerResource {
Alias = "extra"
};
pipeContainer.Properties.Set <string>("image", "someimage");
pipeContainerSidecar.Properties.Set <string>("image", "someimage");
pipeContainerExtra.Properties.Set <string>("image", "someimage");
// Arrange: Create a job request message.
TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
TimelineReference timeline = new TimelineReference();
JobEnvironment environment = new JobEnvironment();
environment.SystemConnection = new ServiceEndpoint();
List <Pipelines.JobStep> steps = new List <Pipelines.JobStep>();
steps.Add(new Pipelines.TaskStep
{
Reference = new Pipelines.TaskStepDefinitionReference()
});
var resources = new Pipelines.JobResources();
resources.Containers.Add(pipeContainer);
resources.Containers.Add(pipeContainerSidecar);
resources.Containers.Add(pipeContainerExtra);
Guid JobId = Guid.NewGuid();
string jobName = "some job name";
var sidecarContainers = new Dictionary <string, string>();
sidecarContainers.Add("sidecar", "sidecar");
var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, null, sidecarContainers,
new Dictionary <string, VariableValue>(), new List <MaskHint>(), resources, new Pipelines.WorkspaceOptions(), steps);
// Arrange: Setup command manager
var pagingLogger = new Mock <IPagingLogger>();
hc.EnqueueInstance(pagingLogger.Object);
// Act.
ec.InitializeJob(jobRequest, CancellationToken.None);
// Assert.
Assert.Equal(2, ec.Containers.Count());
Assert.Equal(1, ec.SidecarContainers.Count());
Assert.False(ec.SidecarContainers.First().IsJobContainer);
}
}
public ContainerInfo(Pipelines.ContainerResource container)
{
this.ContainerName = container.Name;
string containerImage = container.Properties.Get <string>("image");
ArgUtil.NotNullOrEmpty(containerImage, nameof(containerImage));
this.ContainerImage = containerImage;
this.ContainerDisplayName = $"{container.Name}_{Pipelines.Validation.NameValidation.Sanitize(containerImage)}";
this.ContainerRegistryEndpoint = container.Endpoint.Id;
this.ContainerCreateOptions = container.Properties.Get <string>("options");
this.SkipContainerImagePull = container.Properties.Get <bool>("localimage");
this.ContainerEnvironmentVariables = container.Environment;
}
public void StepTarget_RestrictedCommands_Host()
{
using (TestHostContext hc = CreateTestContext())
using (var ec = new Agent.Worker.ExecutionContext())
{
ec.Initialize(hc);
var pipeContainer = new Pipelines.ContainerResource {
Alias = "container"
};
pipeContainer.Properties.Set <string>("image", "someimage");
// Arrange: Create a job request message.
TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference();
TimelineReference timeline = new TimelineReference();
JobEnvironment environment = new JobEnvironment();
environment.SystemConnection = new ServiceEndpoint();
List <Pipelines.JobStep> steps = new List <Pipelines.JobStep>();
steps.Add(new Pipelines.TaskStep
{
Target = new Pipelines.StepTarget
{
Target = "host",
Commands = "restricted"
},
Reference = new Pipelines.TaskStepDefinitionReference()
});
var resources = new Pipelines.JobResources();
resources.Containers.Add(pipeContainer);
Guid JobId = Guid.NewGuid();
string jobName = "some job name";
var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, null, new Dictionary <string, string>(),
new Dictionary <string, VariableValue>(), new List <MaskHint>(), resources, new Pipelines.WorkspaceOptions(), steps);
// Arrange: Setup command manager
var commandMock = new Mock <IWorkerCommandManager>();
hc.SetSingleton(commandMock.Object);
var pagingLogger = new Mock <IPagingLogger>();
hc.EnqueueInstance(pagingLogger.Object);
// Act.
ec.InitializeJob(jobRequest, CancellationToken.None);
ec.SetStepTarget(steps[0].Target);
// Assert.
Assert.IsType <HostInfo>(ec.StepTarget());
commandMock.Verify(x => x.SetCommandRestrictionPolicy(It.IsAny <AttributeBasedWorkerCommandRestrictionPolicy>()));
}
}
public void DefaultContainerInfoMappings()
{
var dockerContainer = new Pipelines.ContainerResource()
{
Alias = "vsts_container_preview",
Image = "foo"
};
using (TestHostContext hc = CreateTestContext())
{
ContainerInfo info = hc.CreateContainerInfo(dockerContainer, isJobContainer: false);
Assert.True(info.TranslateToContainerPath(hc.GetDirectory(WellKnownDirectory.Tools)).EndsWith($"{Path.DirectorySeparatorChar}__t"), "Tools directory maps");
Assert.True(info.TranslateToContainerPath(hc.GetDirectory(WellKnownDirectory.Work)).EndsWith($"{Path.DirectorySeparatorChar}__w"), "Work directory maps");
Assert.True(info.TranslateToContainerPath(hc.GetDirectory(WellKnownDirectory.Root)).EndsWith($"{Path.DirectorySeparatorChar}__a"), "Root directory maps");
}
}
public ContainerInfo(IHostContext hostContext, Pipelines.ContainerResource container, Boolean isJobContainer = true)
{
this.ContainerName = container.Alias;
string containerImage = container.Properties.Get <string>("image");
ArgUtil.NotNullOrEmpty(containerImage, nameof(containerImage));
this.ContainerImage = containerImage;
this.ContainerDisplayName = $"{container.Alias}_{Pipelines.Validation.NameValidation.Sanitize(containerImage)}_{Guid.NewGuid().ToString("N").Substring(0, 6)}";
this.ContainerRegistryEndpoint = container.Endpoint?.Id ?? Guid.Empty;
this.ContainerCreateOptions = container.Properties.Get <string>("options");
this.SkipContainerImagePull = container.Properties.Get <bool>("localimage");
_environmentVariables = container.Environment;
this.ContainerCommand = container.Properties.Get <string>("command", defaultValue: "");
this.IsJobContainer = isJobContainer;
#if OS_WINDOWS
_pathMappings[hostContext.GetDirectory(WellKnownDirectory.Tools)] = "C:\\__t"; // Tool cache folder may come from ENV, so we need a unique folder to avoid collision
_pathMappings[hostContext.GetDirectory(WellKnownDirectory.Work)] = "C:\\__w";
_pathMappings[hostContext.GetDirectory(WellKnownDirectory.Root)] = "C:\\__a";
// add -v '\\.\pipe\docker_engine:\\.\pipe\docker_engine' when they are available (17.09)
#else
_pathMappings[hostContext.GetDirectory(WellKnownDirectory.Tools)] = "/__t"; // Tool cache folder may come from ENV, so we need a unique folder to avoid collision
_pathMappings[hostContext.GetDirectory(WellKnownDirectory.Work)] = "/__w";
_pathMappings[hostContext.GetDirectory(WellKnownDirectory.Root)] = "/__a";
if (this.IsJobContainer)
{
this.MountVolumes.Add(new MountVolume("/var/run/docker.sock", "/var/run/docker.sock"));
}
#endif
if (container.Ports?.Count > 0)
{
foreach (var port in container.Ports)
{
UserPortMappings[port] = port;
}
}
if (container.Volumes?.Count > 0)
{
foreach (var volume in container.Volumes)
{
UserMountVolumes[volume] = volume;
}
}
}
public void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token)
{
// Validation
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Resources, nameof(message.Resources));
ArgUtil.NotNull(message.Variables, nameof(message.Variables));
ArgUtil.NotNull(message.Plan, nameof(message.Plan));
_cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token);
// Features
Features = ApiUtil.GetFeatures(message.Plan);
// Endpoints
Endpoints = message.Resources.Endpoints;
// SecureFiles
SecureFiles = message.Resources.SecureFiles;
// Variables (constructor performs initial recursive expansion)
List <string> warnings;
Variables = new Variables(HostContext, message.Variables, out warnings);
// Prepend Path
PrependPath = new List <string>();
// Docker
string imageName = Variables.Get("_PREVIEW_VSTS_DOCKER_IMAGE");
if (string.IsNullOrEmpty(imageName))
{
imageName = Environment.GetEnvironmentVariable("_PREVIEW_VSTS_DOCKER_IMAGE");
}
if (!string.IsNullOrEmpty(imageName) &&
string.IsNullOrEmpty(message.JobContainer))
{
var dockerContainer = new Pipelines.ContainerResource()
{
Name = "vsts_container_preview"
};
dockerContainer.Properties.Set("image", imageName);
Container = new ContainerInfo(dockerContainer);
}
else if (!string.IsNullOrEmpty(message.JobContainer))
{
Container = new ContainerInfo(message.Resources.Containers.Single(x => string.Equals(x.Name, message.JobContainer, StringComparison.OrdinalIgnoreCase)));
}
else
{
Container = null;
}
// Proxy variables
var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress))
{
Variables.Set(Constants.Variables.Agent.ProxyUrl, agentWebProxy.ProxyAddress);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY", string.Empty);
if (!string.IsNullOrEmpty(agentWebProxy.ProxyUsername))
{
Variables.Set(Constants.Variables.Agent.ProxyUsername, agentWebProxy.ProxyUsername);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_USERNAME", string.Empty);
}
if (!string.IsNullOrEmpty(agentWebProxy.ProxyPassword))
{
Variables.Set(Constants.Variables.Agent.ProxyPassword, agentWebProxy.ProxyPassword, true);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_PASSWORD", string.Empty);
}
if (agentWebProxy.ProxyBypassList.Count > 0)
{
Variables.Set(Constants.Variables.Agent.ProxyBypassList, JsonUtility.ToString(agentWebProxy.ProxyBypassList));
}
}
// Certificate variables
var agentCert = HostContext.GetService <IAgentCertificateManager>();
if (agentCert.SkipServerCertificateValidation)
{
Variables.Set(Constants.Variables.Agent.SslSkipCertValidation, bool.TrueString);
}
if (!string.IsNullOrEmpty(agentCert.CACertificateFile))
{
Variables.Set(Constants.Variables.Agent.SslCAInfo, agentCert.CACertificateFile);
}
if (!string.IsNullOrEmpty(agentCert.ClientCertificateFile) &&
!string.IsNullOrEmpty(agentCert.ClientCertificatePrivateKeyFile) &&
!string.IsNullOrEmpty(agentCert.ClientCertificateArchiveFile))
{
Variables.Set(Constants.Variables.Agent.SslClientCert, agentCert.ClientCertificateFile);
Variables.Set(Constants.Variables.Agent.SslClientCertKey, agentCert.ClientCertificatePrivateKeyFile);
Variables.Set(Constants.Variables.Agent.SslClientCertArchive, agentCert.ClientCertificateArchiveFile);
if (!string.IsNullOrEmpty(agentCert.ClientCertificatePassword))
{
Variables.Set(Constants.Variables.Agent.SslClientCertPassword, agentCert.ClientCertificatePassword, true);
}
}
// Job timeline record.
InitializeTimelineRecord(
timelineId: message.Timeline.Id,
timelineRecordId: message.JobId,
parentTimelineRecordId: null,
recordType: ExecutionContextType.Job,
displayName: message.JobDisplayName,
refName: message.JobName,
order: null); // The job timeline record's order is set by server.
// Logger (must be initialized before writing warnings).
_logger = HostContext.CreateService <IPagingLogger>();
_logger.Setup(_mainTimelineId, _record.Id);
// Log warnings from recursive variable expansion.
warnings?.ForEach(x => this.Warning(x));
// Verbosity (from system.debug).
WriteDebug = Variables.System_Debug ?? false;
// Hook up JobServerQueueThrottling event, we will log warning on server tarpit.
_jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived;
}
public void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token)
{
// Validation
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Resources, nameof(message.Resources));
ArgUtil.NotNull(message.Variables, nameof(message.Variables));
ArgUtil.NotNull(message.Plan, nameof(message.Plan));
_cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token);
// Features
Features = PlanUtil.GetFeatures(message.Plan);
// Endpoints
Endpoints = message.Resources.Endpoints;
// SecureFiles
SecureFiles = message.Resources.SecureFiles;
// Repositories
Repositories = message.Resources.Repositories;
// JobSettings
JobSettings = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
JobSettings[WellKnownJobSettings.HasMultipleCheckouts] = message.Steps?.Where(x => Pipelines.PipelineConstants.IsCheckoutTask(x)).Count() > 1 ? Boolean.TrueString : Boolean.FalseString;
// Variables (constructor performs initial recursive expansion)
List <string> warnings;
Variables = new Variables(HostContext, message.Variables, out warnings);
// Prepend Path
PrependPath = new List <string>();
// Docker (JobContainer)
string imageName = Variables.Get("_PREVIEW_VSTS_DOCKER_IMAGE");
if (string.IsNullOrEmpty(imageName))
{
imageName = Environment.GetEnvironmentVariable("_PREVIEW_VSTS_DOCKER_IMAGE");
}
if (!string.IsNullOrEmpty(imageName) &&
string.IsNullOrEmpty(message.JobContainer))
{
var dockerContainer = new Pipelines.ContainerResource()
{
Alias = "vsts_container_preview"
};
dockerContainer.Properties.Set("image", imageName);
Container = HostContext.CreateContainerInfo(dockerContainer);
}
else if (!string.IsNullOrEmpty(message.JobContainer))
{
Container = HostContext.CreateContainerInfo(message.Resources.Containers.Single(x => string.Equals(x.Alias, message.JobContainer, StringComparison.OrdinalIgnoreCase)));
}
else
{
Container = null;
}
if (Container != null)
{
Container.ImageOSChanged += HandleContainerImageOSChange;
}
// Docker (Sidecar Containers)
SidecarContainers = new List <ContainerInfo>();
foreach (var sidecar in message.JobSidecarContainers)
{
var networkAlias = sidecar.Key;
var containerResourceAlias = sidecar.Value;
var containerResource = message.Resources.Containers.Single(c => string.Equals(c.Alias, containerResourceAlias, StringComparison.OrdinalIgnoreCase));
ContainerInfo containerInfo = HostContext.CreateContainerInfo(containerResource, isJobContainer: false);
containerInfo.ContainerNetworkAlias = networkAlias;
SidecarContainers.Add(containerInfo);
}
// Proxy variables
var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress))
{
Variables.Set(Constants.Variables.Agent.ProxyUrl, agentWebProxy.ProxyAddress);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY", string.Empty);
if (!string.IsNullOrEmpty(agentWebProxy.ProxyUsername))
{
Variables.Set(Constants.Variables.Agent.ProxyUsername, agentWebProxy.ProxyUsername);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_USERNAME", string.Empty);
}
if (!string.IsNullOrEmpty(agentWebProxy.ProxyPassword))
{
Variables.Set(Constants.Variables.Agent.ProxyPassword, agentWebProxy.ProxyPassword, true);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_PASSWORD", string.Empty);
}
if (agentWebProxy.ProxyBypassList.Count > 0)
{
Variables.Set(Constants.Variables.Agent.ProxyBypassList, JsonUtility.ToString(agentWebProxy.ProxyBypassList));
}
}
// Certificate variables
var agentCert = HostContext.GetService <IAgentCertificateManager>();
if (agentCert.SkipServerCertificateValidation)
{
Variables.Set(Constants.Variables.Agent.SslSkipCertValidation, bool.TrueString);
}
if (!string.IsNullOrEmpty(agentCert.CACertificateFile))
{
Variables.Set(Constants.Variables.Agent.SslCAInfo, agentCert.CACertificateFile);
}
if (!string.IsNullOrEmpty(agentCert.ClientCertificateFile) &&
!string.IsNullOrEmpty(agentCert.ClientCertificatePrivateKeyFile) &&
!string.IsNullOrEmpty(agentCert.ClientCertificateArchiveFile))
{
Variables.Set(Constants.Variables.Agent.SslClientCert, agentCert.ClientCertificateFile);
Variables.Set(Constants.Variables.Agent.SslClientCertKey, agentCert.ClientCertificatePrivateKeyFile);
Variables.Set(Constants.Variables.Agent.SslClientCertArchive, agentCert.ClientCertificateArchiveFile);
if (!string.IsNullOrEmpty(agentCert.ClientCertificatePassword))
{
Variables.Set(Constants.Variables.Agent.SslClientCertPassword, agentCert.ClientCertificatePassword, true);
}
}
// Runtime option variables
var runtimeOptions = HostContext.GetService <IConfigurationStore>().GetAgentRuntimeOptions();
if (runtimeOptions != null)
{
if (PlatformUtil.RunningOnWindows && runtimeOptions.GitUseSecureChannel)
{
Variables.Set(Constants.Variables.Agent.GitUseSChannel, runtimeOptions.GitUseSecureChannel.ToString());
}
}
// Job timeline record.
InitializeTimelineRecord(
timelineId: message.Timeline.Id,
timelineRecordId: message.JobId,
parentTimelineRecordId: null,
recordType: ExecutionContextType.Job,
displayName: message.JobDisplayName,
refName: message.JobName,
order: null); // The job timeline record's order is set by server.
// Logger (must be initialized before writing warnings).
_logger = HostContext.CreateService <IPagingLogger>();
_logger.Setup(_mainTimelineId, _record.Id);
// Log warnings from recursive variable expansion.
warnings?.ForEach(x => this.Warning(x));
// Verbosity (from system.debug).
WriteDebug = Variables.System_Debug ?? false;
// Hook up JobServerQueueThrottling event, we will log warning on server tarpit.
_jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived;
}
public void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token)
{
// Validation
Trace.Entering();
ArgUtil.NotNull(message, nameof(message));
ArgUtil.NotNull(message.Resources, nameof(message.Resources));
ArgUtil.NotNull(message.Variables, nameof(message.Variables));
ArgUtil.NotNull(message.Plan, nameof(message.Plan));
_cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token);
// Features
Features = PlanUtil.GetFeatures(message.Plan);
// Endpoints
Endpoints = message.Resources.Endpoints;
// SecureFiles
SecureFiles = message.Resources.SecureFiles;
// Repositories
Repositories = message.Resources.Repositories;
// JobSettings
var checkouts = message.Steps?.Where(x => Pipelines.PipelineConstants.IsCheckoutTask(x)).ToList();
JobSettings = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
JobSettings[WellKnownJobSettings.HasMultipleCheckouts] = Boolean.FalseString;
if (checkouts != null && checkouts.Count > 0)
{
JobSettings[WellKnownJobSettings.HasMultipleCheckouts] = checkouts.Count > 1 ? Boolean.TrueString : Boolean.FalseString;
var firstCheckout = checkouts.First() as Pipelines.TaskStep;
if (firstCheckout != null && Repositories != null && firstCheckout.Inputs.TryGetValue(Pipelines.PipelineConstants.CheckoutTaskInputs.Repository, out string repoAlias))
{
JobSettings[WellKnownJobSettings.FirstRepositoryCheckedOut] = repoAlias;
var repo = Repositories.Find(r => String.Equals(r.Alias, repoAlias, StringComparison.OrdinalIgnoreCase));
if (repo != null)
{
repo.Properties.Set <bool>(RepositoryUtil.IsPrimaryRepository, true);
}
}
}
// Variables (constructor performs initial recursive expansion)
List <string> warnings;
Variables = new Variables(HostContext, message.Variables, out warnings);
Variables.StringTranslator = TranslatePathForStepTarget;
if (Variables.GetBoolean("agent.useWorkspaceId") == true)
{
try
{
// We need an identifier that represents which repos make up the workspace.
// This allows similar jobs in the same definition to reuse that workspace and other jobs to have their own.
JobSettings[WellKnownJobSettings.WorkspaceIdentifier] = GetWorkspaceIdentifier(message);
}
catch (Exception ex)
{
Trace.Warning($"Unable to generate workspace ID: {ex.Message}");
}
}
// Prepend Path
PrependPath = new List <string>();
// Docker (JobContainer)
string imageName = Variables.Get("_PREVIEW_VSTS_DOCKER_IMAGE");
if (string.IsNullOrEmpty(imageName))
{
imageName = Environment.GetEnvironmentVariable("_PREVIEW_VSTS_DOCKER_IMAGE");
}
Containers = new List <ContainerInfo>();
_defaultStepTarget = null;
_currentStepTarget = null;
if (!string.IsNullOrEmpty(imageName) &&
string.IsNullOrEmpty(message.JobContainer))
{
var dockerContainer = new Pipelines.ContainerResource()
{
Alias = "vsts_container_preview"
};
dockerContainer.Properties.Set("image", imageName);
var defaultJobContainer = HostContext.CreateContainerInfo(dockerContainer);
_defaultStepTarget = defaultJobContainer;
Containers.Add(defaultJobContainer);
}
else if (!string.IsNullOrEmpty(message.JobContainer))
{
var defaultJobContainer = HostContext.CreateContainerInfo(message.Resources.Containers.Single(x => string.Equals(x.Alias, message.JobContainer, StringComparison.OrdinalIgnoreCase)));
_defaultStepTarget = defaultJobContainer;
Containers.Add(defaultJobContainer);
}
else
{
_defaultStepTarget = new HostInfo();
}
// Include other step containers
var sidecarContainers = new HashSet <string>(message.JobSidecarContainers.Values, StringComparer.OrdinalIgnoreCase);
foreach (var container in message.Resources.Containers.Where(x =>
!string.Equals(x.Alias, message.JobContainer, StringComparison.OrdinalIgnoreCase) && !sidecarContainers.Contains(x.Alias)))
{
Containers.Add(HostContext.CreateContainerInfo(container));
}
// Docker (Sidecar Containers)
SidecarContainers = new List <ContainerInfo>();
foreach (var sidecar in message.JobSidecarContainers)
{
var networkAlias = sidecar.Key;
var containerResourceAlias = sidecar.Value;
var containerResource = message.Resources.Containers.Single(c => string.Equals(c.Alias, containerResourceAlias, StringComparison.OrdinalIgnoreCase));
ContainerInfo containerInfo = HostContext.CreateContainerInfo(containerResource, isJobContainer: false);
containerInfo.ContainerNetworkAlias = networkAlias;
SidecarContainers.Add(containerInfo);
}
// Proxy variables
var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>();
if (!string.IsNullOrEmpty(agentWebProxy.ProxyAddress))
{
Variables.Set(Constants.Variables.Agent.ProxyUrl, agentWebProxy.ProxyAddress);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY", string.Empty);
if (!string.IsNullOrEmpty(agentWebProxy.ProxyUsername))
{
Variables.Set(Constants.Variables.Agent.ProxyUsername, agentWebProxy.ProxyUsername);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_USERNAME", string.Empty);
}
if (!string.IsNullOrEmpty(agentWebProxy.ProxyPassword))
{
Variables.Set(Constants.Variables.Agent.ProxyPassword, agentWebProxy.ProxyPassword, true);
Environment.SetEnvironmentVariable("VSTS_HTTP_PROXY_PASSWORD", string.Empty);
}
if (agentWebProxy.ProxyBypassList.Count > 0)
{
Variables.Set(Constants.Variables.Agent.ProxyBypassList, JsonUtility.ToString(agentWebProxy.ProxyBypassList));
}
}
// Certificate variables
var agentCert = HostContext.GetService <IAgentCertificateManager>();
if (agentCert.SkipServerCertificateValidation)
{
Variables.Set(Constants.Variables.Agent.SslSkipCertValidation, bool.TrueString);
}
if (!string.IsNullOrEmpty(agentCert.CACertificateFile))
{
Variables.Set(Constants.Variables.Agent.SslCAInfo, agentCert.CACertificateFile);
}
if (!string.IsNullOrEmpty(agentCert.ClientCertificateFile) &&
!string.IsNullOrEmpty(agentCert.ClientCertificatePrivateKeyFile) &&
!string.IsNullOrEmpty(agentCert.ClientCertificateArchiveFile))
{
Variables.Set(Constants.Variables.Agent.SslClientCert, agentCert.ClientCertificateFile);
Variables.Set(Constants.Variables.Agent.SslClientCertKey, agentCert.ClientCertificatePrivateKeyFile);
Variables.Set(Constants.Variables.Agent.SslClientCertArchive, agentCert.ClientCertificateArchiveFile);
if (!string.IsNullOrEmpty(agentCert.ClientCertificatePassword))
{
Variables.Set(Constants.Variables.Agent.SslClientCertPassword, agentCert.ClientCertificatePassword, true);
}
}
// Runtime option variables
var runtimeOptions = HostContext.GetService <IConfigurationStore>().GetAgentRuntimeOptions();
if (runtimeOptions != null)
{
if (PlatformUtil.RunningOnWindows && runtimeOptions.GitUseSecureChannel)
{
Variables.Set(Constants.Variables.Agent.GitUseSChannel, runtimeOptions.GitUseSecureChannel.ToString());
}
}
// Job timeline record.
InitializeTimelineRecord(
timelineId: message.Timeline.Id,
timelineRecordId: message.JobId,
parentTimelineRecordId: null,
recordType: ExecutionContextType.Job,
displayName: message.JobDisplayName,
refName: message.JobName,
order: null); // The job timeline record's order is set by server.
// Logger (must be initialized before writing warnings).
_logger = HostContext.CreateService <IPagingLogger>();
_logger.Setup(_mainTimelineId, _record.Id);
// Log warnings from recursive variable expansion.
warnings?.ForEach(x => this.Warning(x));
// Verbosity (from system.debug).
WriteDebug = Variables.System_Debug ?? false;
// Hook up JobServerQueueThrottling event, we will log warning on server tarpit.
_jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived;
}
public void GeneratePluginExecutionContextContainerInfoTest()
{
var dockerContainer = new Pipelines.ContainerResource()
{
Alias = "vsts_container_preview",
Image = "foo"
};
using (TestHostContext tc = CreateTestContext())
{
Tracing trace = tc.GetTrace();
var agentPluginManager = new AgentPluginManager();
agentPluginManager.Initialize(tc);
var containerInfo = tc.CreateContainerInfo(dockerContainer, isJobContainer: false);
var containerWorkPath = "/__w";
if (TestUtil.IsWindows())
{
containerWorkPath = "C:\\__w";
}
var inputs = new Dictionary <string, string>()
{
{ "input1", "foo" },
{ "input2", containerWorkPath },
{ "input3", tc.GetDirectory(WellKnownDirectory.Work) },
};
var expectedInputs = new Dictionary <string, string>()
{
{ "input1", "foo" },
{ "input2", tc.GetDirectory(WellKnownDirectory.Work) },
{ "input3", tc.GetDirectory(WellKnownDirectory.Work) },
};
var variables = new Dictionary <string, VariableValue>()
{
{ "variable1", "foo" },
{ "variable2", containerWorkPath },
{ "variable3", tc.GetDirectory(WellKnownDirectory.Work) },
};
var expectedVariables = new Dictionary <string, VariableValue>()
{
{ "variable1", "foo" },
{ "variable2", tc.GetDirectory(WellKnownDirectory.Work) },
{ "variable3", tc.GetDirectory(WellKnownDirectory.Work) },
};
var taskVariables = new Dictionary <string, VariableValue>()
{
{ "taskVariable1", "foo" },
{ "taskVariable2", containerWorkPath },
{ "taskVariable3", tc.GetDirectory(WellKnownDirectory.Work) },
};
var expectedTaskVariables = new Dictionary <string, VariableValue>()
{
{ "taskVariable1", "foo" },
{ "taskVariable2", tc.GetDirectory(WellKnownDirectory.Work) },
{ "taskVariable3", tc.GetDirectory(WellKnownDirectory.Work) },
};
var executionContext = CreateTestExecutionContext(tc, stepTarget: containerInfo, variables: variables, taskVariables: taskVariables);
var pluginContext = agentPluginManager.GeneratePluginExecutionContext(executionContext, inputs, executionContext.Variables);
Assert.True(pluginContext != null, "PluginContext for Container Step Target is not null");
Assert.True(expectedInputs.All(e => pluginContext.Inputs.Contains(e)));
Assert.True(expectedVariables.All(e => pluginContext.Variables.Contains(e)));
Assert.True(expectedTaskVariables.All(e => pluginContext.TaskVariables.Contains(e)));
}
}