public async Task UnshelveAsync(string shelveset)
{
ArgUtil.NotNullOrEmpty(shelveset, nameof(shelveset));
await RunCommandAsync(FormatFlags.OmitCollectionUrl, "vc", "unshelve", shelveset);
}
public async Task <int> ExecuteAsync(string workingDirectory,
string fileName,
string arguments,
IDictionary <string, string> environment,
bool requireExitCodeZero,
Encoding outputEncoding,
bool killProcessOnCancel,
bool inheritConsoleHandler,
CancellationToken cancellationToken)
{
// make sure container exist.
ArgUtil.NotNull(Container, nameof(Container));
ArgUtil.NotNullOrEmpty(Container.ContainerId, nameof(Container.ContainerId));
var dockerManger = HostContext.GetService <IDockerCommandManager>();
string containerEnginePath = dockerManger.DockerPath;
ContainerStandardInPayload payload = new ContainerStandardInPayload()
{
ExecutionHandler = fileName,
ExecutionHandlerWorkingDirectory = workingDirectory,
ExecutionHandlerArguments = arguments,
ExecutionHandlerEnvironment = environment,
ExecutionHandlerPrependPath = PrependPath
};
// copy the intermediate script (containerHandlerInvoker.js) into Agent_TempDirectory
// Background:
// We rely on environment variables to send task execution information from agent to task execution engine (node/powershell)
// Those task execution information will include all the variables and secrets customer has.
// The only way to pass environment variables to `docker exec` is through command line arguments, ex: `docker exec -e myenv=myvalue -e mysecert=mysecretvalue ...`
// Since command execution may get log into system event log which might cause secret leaking.
// We use this intermediate script to read everything from STDIN, then launch the task execution engine (node/powershell) and redirect STDOUT/STDERR
string tempDir = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), Constants.Path.TempDirectory);
File.Copy(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), "containerHandlerInvoker.js.template"), Path.Combine(tempDir, "containerHandlerInvoker.js"), true);
string node;
if (!string.IsNullOrEmpty(Container.ContainerBringNodePath))
{
node = Container.ContainerBringNodePath;
}
else
{
node = Container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node", "bin", $"node{IOUtil.ExeExtension}"));
}
string entryScript = Container.TranslateToContainerPath(Path.Combine(tempDir, "containerHandlerInvoker.js"));
string containerExecutionArgs;
if (PlatformUtil.RunningOnWindows)
{
containerExecutionArgs = $"exec -i {Container.ContainerId} {node} {entryScript}";
}
else
{
containerExecutionArgs = $"exec -i -u {Container.CurrentUserId} {Container.ContainerId} {node} {entryScript}";
}
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += OutputDataReceived;
processInvoker.ErrorDataReceived += ErrorDataReceived;
// Let .NET choose the default encoding, except on Windows.
outputEncoding = null;
if (PlatformUtil.RunningOnWindows)
{
// It appears that node.exe outputs UTF8 when not in TTY mode.
outputEncoding = Encoding.UTF8;
}
var redirectStandardIn = new InputQueue <string>();
redirectStandardIn.Enqueue(JsonUtility.ToString(payload));
return(await processInvoker.ExecuteAsync(workingDirectory : HostContext.GetDirectory(WellKnownDirectory.Work),
fileName : containerEnginePath,
arguments : containerExecutionArgs,
environment : null,
requireExitCodeZero : requireExitCodeZero,
outputEncoding : outputEncoding,
killProcessOnCancel : killProcessOnCancel,
redirectStandardIn : redirectStandardIn,
inheritConsoleHandler : inheritConsoleHandler,
cancellationToken : cancellationToken));
}
}
private async Task StartContainerAsync(IExecutionContext executionContext, ContainerInfo container)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(container, nameof(container));
ArgUtil.NotNullOrEmpty(container.ContainerImage, nameof(container.ContainerImage));
Trace.Info($"Container name: {container.ContainerName}");
Trace.Info($"Container image: {container.ContainerImage}");
Trace.Info($"Container registry: {container.ContainerRegistryEndpoint.ToString()}");
Trace.Info($"Container options: {container.ContainerCreateOptions}");
Trace.Info($"Skip container image pull: {container.SkipContainerImagePull}");
foreach (var port in container.UserPortMappings)
{
Trace.Info($"User provided port: {port.Value}");
}
foreach (var volume in container.UserMountVolumes)
{
Trace.Info($"User provided volume: {volume.Value}");
}
if (container.ImageOS != PlatformUtil.OS.Windows)
{
string defaultWorkingDirectory = executionContext.Variables.Get(Constants.Variables.System.DefaultWorkingDirectory);
defaultWorkingDirectory = container.TranslateContainerPathForImageOS(PlatformUtil.HostOS, container.TranslateToContainerPath(defaultWorkingDirectory));
if (string.IsNullOrEmpty(defaultWorkingDirectory))
{
throw new NotSupportedException(StringUtil.Loc("ContainerJobRequireSystemDefaultWorkDir"));
}
string workingDirectory = IOUtil.GetDirectoryName(defaultWorkingDirectory.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), container.ImageOS);
string mountWorkingDirectory = container.TranslateToHostPath(workingDirectory);
executionContext.Debug($"Default Working Directory {defaultWorkingDirectory}");
executionContext.Debug($"Working Directory {workingDirectory}");
executionContext.Debug($"Mount Working Directory {mountWorkingDirectory}");
if (!string.IsNullOrEmpty(workingDirectory))
{
container.MountVolumes.Add(new MountVolume(mountWorkingDirectory, workingDirectory));
}
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Temp), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Temp))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tasks), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tasks))));
}
else
{
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Work), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Work))));
}
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools))));
bool externalReadOnly = container.ImageOS != PlatformUtil.OS.Windows; // This code was refactored to use PlatformUtils. The previous implementation did not have the externals directory mounted read-only for Windows.
// That seems wrong, but to prevent any potential backwards compatibility issues, we are keeping the same logic
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)), externalReadOnly));
if (container.ImageOS != PlatformUtil.OS.Windows)
{
// Ensure .taskkey file exist so we can mount it.
string taskKeyFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), ".taskkey");
if (!File.Exists(taskKeyFile))
{
File.WriteAllText(taskKeyFile, string.Empty);
}
container.MountVolumes.Add(new MountVolume(taskKeyFile, container.TranslateToContainerPath(taskKeyFile)));
}
if (container.IsJobContainer)
{
// See if this container brings its own Node.js
container.CustomNodePath = await _dockerManger.DockerInspect(context : executionContext,
dockerObject : container.ContainerImage,
options : $"--format=\"{{{{index .Config.Labels \\\"{_nodeJsPathLabel}\\\"}}}}\"");
string node;
if (!string.IsNullOrEmpty(container.CustomNodePath))
{
node = container.CustomNodePath;
}
else
{
node = container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node", "bin", $"node{IOUtil.ExeExtension}"));
// if on Mac OS X, require container to have node
if (PlatformUtil.RunningOnMacOS)
{
container.CustomNodePath = "node";
node = container.CustomNodePath;
}
// if running on Windows, and attempting to run linux container, require container to have node
else if (PlatformUtil.RunningOnWindows && container.ImageOS == PlatformUtil.OS.Linux)
{
container.CustomNodePath = "node";
node = container.CustomNodePath;
}
}
string sleepCommand = $"\"{node}\" -e \"setInterval(function(){{}}, 24 * 60 * 60 * 1000);\"";
container.ContainerCommand = sleepCommand;
}
container.ContainerId = await _dockerManger.DockerCreate(executionContext, container);
ArgUtil.NotNullOrEmpty(container.ContainerId, nameof(container.ContainerId));
if (container.IsJobContainer)
{
executionContext.Variables.Set(Constants.Variables.Agent.ContainerId, container.ContainerId);
}
// Start container
int startExitCode = await _dockerManger.DockerStart(executionContext, container.ContainerId);
if (startExitCode != 0)
{
throw new InvalidOperationException($"Docker start fail with exit code {startExitCode}");
}
try
{
// Make sure container is up and running
var psOutputs = await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --filter status=running --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\"");
if (psOutputs.FirstOrDefault(x => !string.IsNullOrEmpty(x))?.StartsWith(container.ContainerId) != true)
{
// container is not up and running, pull docker log for this container.
await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\"");
int logsExitCode = await _dockerManger.DockerLogs(executionContext, container.ContainerId);
if (logsExitCode != 0)
{
executionContext.Warning($"Docker logs fail with exit code {logsExitCode}");
}
executionContext.Warning($"Docker container {container.ContainerId} is not in running state.");
}
}
catch (Exception ex)
{
// pull container log is best effort.
Trace.Error("Catch exception when check container log and container status.");
Trace.Error(ex);
}
// Get port mappings of running container
if (!container.IsJobContainer)
{
container.AddPortMappings(await _dockerManger.DockerPort(executionContext, container.ContainerId));
foreach (var port in container.PortMappings)
{
executionContext.Variables.Set(
$"{Constants.Variables.Agent.ServicePortPrefix}.{container.ContainerNetworkAlias}.ports.{port.ContainerPort}",
$"{port.HostPort}");
}
}
if (!PlatformUtil.RunningOnWindows)
{
if (container.IsJobContainer)
{
// Ensure bash exist in the image
int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"sh -c \"command -v bash\"");
if (execWhichBashExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execWhichBashExitCode}");
}
// Get current username
container.CurrentUserName = (await ExecuteCommandAsync(executionContext, "whoami", string.Empty)).FirstOrDefault();
ArgUtil.NotNullOrEmpty(container.CurrentUserName, nameof(container.CurrentUserName));
// Get current userId
container.CurrentUserId = (await ExecuteCommandAsync(executionContext, "id", $"-u {container.CurrentUserName}")).FirstOrDefault();
ArgUtil.NotNullOrEmpty(container.CurrentUserId, nameof(container.CurrentUserId));
executionContext.Output(StringUtil.Loc("CreateUserWithSameUIDInsideContainer", container.CurrentUserId));
// Create an user with same uid as the agent run as user inside the container.
// All command execute in docker will run as Root by default,
// this will cause the agent on the host machine doesn't have permission to any new file/folder created inside the container.
// So, we create a user account with same UID inside the container and let all docker exec command run as that user.
string containerUserName = string.Empty;
// We need to find out whether there is a user with same UID inside the container
List <string> userNames = new List <string>();
int execGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"getent passwd {container.CurrentUserId} | cut -d: -f1 \"", userNames);
if (execGrepExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGrepExitCode}");
}
if (userNames.Count > 0)
{
// check all potential username that might match the UID.
foreach (string username in userNames)
{
int execIdExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"id -u {username}");
if (execIdExitCode == 0)
{
containerUserName = username;
break;
}
}
}
// Create a new user with same UID
if (string.IsNullOrEmpty(containerUserName))
{
containerUserName = $"{container.CurrentUserName}_azpcontainer";
int execUseraddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"useradd -m -u {container.CurrentUserId} {containerUserName}");
if (execUseraddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execUseraddExitCode}");
}
}
executionContext.Output(StringUtil.Loc("GrantContainerUserSUDOPrivilege", containerUserName));
// Create a new group for giving sudo permission
int execGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd azure_pipelines_sudo");
if (execGroupaddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGroupaddExitCode}");
}
// Add the new created user to the new created sudo group.
int execUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G azure_pipelines_sudo {containerUserName}");
if (execUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execUsermodExitCode}");
}
// Allow the new sudo group run any sudo command without providing password.
int execEchoExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"su -c \"echo '%azure_pipelines_sudo ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers\"");
if (execUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execEchoExitCode}");
}
if (AgentKnobs.SetupDockerGroup.GetValue(executionContext).AsBoolean())
{
executionContext.Output(StringUtil.Loc("AllowContainerUserRunDocker", containerUserName));
// Get docker.sock group id on Host
string statFormatOption = "-c %g";
if (PlatformUtil.RunningOnMacOS)
{
statFormatOption = "-f %g";
}
string dockerSockGroupId = (await ExecuteCommandAsync(executionContext, "stat", $"{statFormatOption} /var/run/docker.sock")).FirstOrDefault();
// We need to find out whether there is a group with same GID inside the container
string existingGroupName = null;
List <string> groupsOutput = new List <string>();
int execGroupGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"cat /etc/group\"", groupsOutput);
if (execGroupGrepExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGroupGrepExitCode}");
}
if (groupsOutput.Count > 0)
{
// check all potential groups that might match the GID.
foreach (string groupOutput in groupsOutput)
{
if (!string.IsNullOrEmpty(groupOutput))
{
var groupSegments = groupOutput.Split(':');
if (groupSegments.Length != 4)
{
Trace.Warning($"Unexpected output from /etc/group: '{groupOutput}'");
}
else
{
// the output of /etc/group should looks like `group:x:gid:`
var groupName = groupSegments[0];
var groupId = groupSegments[2];
if (string.Equals(dockerSockGroupId, groupId))
{
existingGroupName = groupName;
break;
}
}
}
}
}
if (string.IsNullOrEmpty(existingGroupName))
{
// create a new group with same gid
existingGroupName = "azure_pipelines_docker";
int execDockerGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd -g {dockerSockGroupId} azure_pipelines_docker");
if (execDockerGroupaddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execDockerGroupaddExitCode}");
}
}
// Add the new created user to the docker socket group.
int execGroupUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G {existingGroupName} {containerUserName}");
if (execGroupUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGroupUsermodExitCode}");
}
// if path to node is just 'node', with no path, let's make sure it is actually there
if (string.Equals(container.CustomNodePath, "node", StringComparison.OrdinalIgnoreCase))
{
List <string> nodeVersionOutput = new List <string>();
int execNodeVersionExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"node -v\"", nodeVersionOutput);
if (execNodeVersionExitCode != 0)
{
throw new InvalidOperationException($"Unable to get node version on container {container.ContainerId}. Got exit code {execNodeVersionExitCode} from docker exec");
}
if (nodeVersionOutput.Count > 0)
{
executionContext.Output($"Detected Node Version: {nodeVersionOutput[0]}");
Trace.Info($"Using node version {nodeVersionOutput[0]} in container {container.ContainerId}");
}
else
{
throw new InvalidOperationException($"Unable to get node version on container {container.ContainerId}. No output from node -v");
}
}
}
}
}
}
protected override async Task ProcessCommandInternalAsync(
AgentTaskPluginExecutionContext context,
CancellationToken token)
{
ArgUtil.NotNull(context, nameof(context));
string artifactName = context.GetInput(ArtifactEventProperties.ArtifactName, required: false);
string branchName = context.GetInput(ArtifactEventProperties.BranchName, required: false);
string pipelineDefinition = context.GetInput(ArtifactEventProperties.PipelineDefinition, required: false);
string sourceRun = context.GetInput(ArtifactEventProperties.SourceRun, required: true);
string pipelineTriggering = context.GetInput(ArtifactEventProperties.PipelineTriggering, required: false);
string pipelineVersionToDownload = context.GetInput(ArtifactEventProperties.PipelineVersionToDownload, required: false);
string targetPath = context.GetInput(DownloadPath, required: true);
string environmentBuildId = context.Variables.GetValueOrDefault(BuildVariables.BuildId)?.Value ?? string.Empty; // BuildID provided by environment.
string itemPattern = context.GetInput(ArtifactEventProperties.ItemPattern, required: false);
string projectName = context.GetInput(ArtifactEventProperties.Project, required: false);
string tags = context.GetInput(ArtifactEventProperties.Tags, required: false);
string allowPartiallySucceededBuilds = context.GetInput(ArtifactEventProperties.AllowPartiallySucceededBuilds, required: false);
string allowFailedBuilds = context.GetInput(ArtifactEventProperties.AllowFailedBuilds, required: false);
string allowCanceledBuilds = context.GetInput(ArtifactEventProperties.AllowCanceledBuilds, required: false);
string userSpecifiedRunId = context.GetInput(RunId, required: false);
string defaultWorkingDirectory = context.Variables.GetValueOrDefault("system.defaultworkingdirectory").Value;
targetPath = Path.IsPathFullyQualified(targetPath) ? targetPath : Path.GetFullPath(Path.Combine(defaultWorkingDirectory, targetPath));
bool onPrem = !String.Equals(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.ServerType)?.Value, "Hosted", StringComparison.OrdinalIgnoreCase);
if (onPrem)
{
throw new InvalidOperationException(StringUtil.Loc("OnPremIsNotSupported"));
}
if (!PipelineArtifactPathHelper.IsValidArtifactName(artifactName))
{
throw new ArgumentException(StringUtil.Loc("ArtifactNameIsNotValid", artifactName));
}
string[] minimatchPatterns = itemPattern.Split(
new[] { "\n" },
StringSplitOptions.RemoveEmptyEntries
);
string[] tagsInput = tags.Split(
new[] { "," },
StringSplitOptions.None
);
if (!bool.TryParse(allowPartiallySucceededBuilds, out var allowPartiallySucceededBuildsBool))
{
allowPartiallySucceededBuildsBool = false;
}
if (!bool.TryParse(allowFailedBuilds, out var allowFailedBuildsBool))
{
allowFailedBuildsBool = false;
}
if (!bool.TryParse(allowCanceledBuilds, out var allowCanceledBuildsBool))
{
allowCanceledBuildsBool = false;
}
var resultFilter = GetResultFilter(allowPartiallySucceededBuildsBool, allowFailedBuildsBool, allowCanceledBuildsBool);
PipelineArtifactServer server = new PipelineArtifactServer(tracer);
PipelineArtifactDownloadParameters downloadParameters;
if (sourceRun == sourceRunCurrent)
{
// TODO: use a constant for project id, which is currently defined in Microsoft.VisualStudio.Services.Agent.Constants.Variables.System.TeamProjectId (Ting)
string projectIdStr = context.Variables.GetValueOrDefault("system.teamProjectId")?.Value;
if (String.IsNullOrEmpty(projectIdStr))
{
throw new ArgumentNullException(StringUtil.Loc("CannotBeNullOrEmpty"), "Project ID");
}
Guid projectId = Guid.Parse(projectIdStr);
ArgUtil.NotEmpty(projectId, nameof(projectId));
int pipelineId = 0;
if (int.TryParse(environmentBuildId, out pipelineId) && pipelineId != 0)
{
OutputBuildInfo(context, pipelineId);
}
else
{
string hostType = context.Variables.GetValueOrDefault("system.hosttype")?.Value;
if (string.Equals(hostType, "Release", StringComparison.OrdinalIgnoreCase) ||
string.Equals(hostType, "DeploymentGroup", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException(StringUtil.Loc("BuildIdIsNotAvailable", hostType ?? string.Empty, hostType ?? string.Empty));
}
else if (!string.Equals(hostType, "Build", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException(StringUtil.Loc("CannotDownloadFromCurrentEnvironment", hostType ?? string.Empty));
}
else
{
// This should not happen since the build id comes from build environment. But a user may override that so we must be careful.
throw new ArgumentException(StringUtil.Loc("BuildIdIsNotValid", environmentBuildId));
}
}
downloadParameters = new PipelineArtifactDownloadParameters
{
ProjectRetrievalOptions = BuildArtifactRetrievalOptions.RetrieveByProjectId,
ProjectId = projectId,
PipelineId = pipelineId,
ArtifactName = artifactName,
TargetDirectory = targetPath,
MinimatchFilters = minimatchPatterns,
MinimatchFilterWithArtifactName = true
};
}
else if (sourceRun == sourceRunSpecific)
{
if (String.IsNullOrEmpty(projectName))
{
throw new ArgumentNullException(StringUtil.Loc("CannotBeNullOrEmpty"), "Project Name");
}
Guid projectId;
bool isProjGuid = Guid.TryParse(projectName, out projectId);
if (!isProjGuid)
{
projectId = await GetProjectIdAsync(context, projectName);
}
// Set the default pipelineId to 0, which is an invalid build id and it has to be reassigned to a valid build id.
int pipelineId = 0;
bool pipelineTriggeringBool = false;
if (bool.TryParse(pipelineTriggering, out pipelineTriggeringBool) && pipelineTriggeringBool)
{
string triggeringPipeline = context.Variables.GetValueOrDefault("build.triggeredBy.buildId")?.Value;
if (!string.IsNullOrEmpty(triggeringPipeline))
{
pipelineId = int.Parse(triggeringPipeline);
}
}
if (pipelineId == 0)
{
if (pipelineVersionToDownload == pipelineVersionToDownloadLatest)
{
pipelineId = await this.GetPipelineIdAsync(context, pipelineDefinition, pipelineVersionToDownload, projectId.ToString(), tagsInput, resultFilter, null, cancellationToken : token);
}
else if (pipelineVersionToDownload == pipelineVersionToDownloadSpecific)
{
bool isPipelineIdNum = Int32.TryParse(userSpecifiedRunId, out pipelineId);
if (!isPipelineIdNum)
{
throw new ArgumentException(StringUtil.Loc("RunIDNotValid", userSpecifiedRunId));
}
}
else if (pipelineVersionToDownload == pipelineVersionToDownloadLatestFromBranch)
{
pipelineId = await this.GetPipelineIdAsync(context, pipelineDefinition, pipelineVersionToDownload, projectId.ToString(), tagsInput, resultFilter, branchName, cancellationToken : token);
}
else
{
throw new InvalidOperationException("Unreachable code!");
}
}
OutputBuildInfo(context, pipelineId);
downloadParameters = new PipelineArtifactDownloadParameters
{
ProjectRetrievalOptions = BuildArtifactRetrievalOptions.RetrieveByProjectName,
ProjectName = projectName,
ProjectId = projectId,
PipelineId = pipelineId,
ArtifactName = artifactName,
TargetDirectory = targetPath,
MinimatchFilters = minimatchPatterns,
MinimatchFilterWithArtifactName = true
};
}
else
{
throw new InvalidOperationException($"Build type '{sourceRun}' is not recognized.");
}
string fullPath = this.CreateDirectoryIfDoesntExist(targetPath);
DownloadOptions downloadOptions;
if (string.IsNullOrEmpty(downloadParameters.ArtifactName))
{
downloadOptions = DownloadOptions.MultiDownload;
}
else
{
downloadOptions = DownloadOptions.SingleDownload;
}
context.Output(StringUtil.Loc("DownloadArtifactTo", targetPath));
await server.DownloadAsyncV2(context, downloadParameters, downloadOptions, token);
context.Output(StringUtil.Loc("DownloadArtifactFinished"));
}
public TrackingConfig PrepareDirectory(
IExecutionContext executionContext,
IList <RepositoryResource> repositories,
WorkspaceOptions workspace)
{
// Validate parameters.
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(executionContext.Variables, nameof(executionContext.Variables));
ArgUtil.NotNull(repositories, nameof(repositories));
var trackingManager = HostContext.GetService <ITrackingManager>();
// Create the tracking config for this execution of the pipeline
var agentSettings = HostContext.GetService <IConfigurationStore>().GetSettings();
var newConfig = trackingManager.Create(executionContext, repositories, ShouldOverrideBuildDirectory(repositories, agentSettings));
// Load the tracking config from the last execution of the pipeline
var existingConfig = trackingManager.LoadExistingTrackingConfig(executionContext);
// If there aren't any major changes, merge the configurations and use the same workspace
if (trackingManager.AreTrackingConfigsCompatible(executionContext, newConfig, existingConfig))
{
newConfig = trackingManager.MergeTrackingConfigs(executionContext, newConfig, existingConfig);
}
else if (existingConfig != null)
{
// If the previous config had different repos, get a new workspace folder and mark the old one for clean up
trackingManager.MarkForGarbageCollection(executionContext, existingConfig);
// If the config file was updated to a new config, we need to delete the legacy artifact/staging directories.
// DeleteDirectory will check for the existence of the folders first.
DeleteDirectory(
executionContext,
description: "legacy artifacts directory",
path: Path.Combine(existingConfig.BuildDirectory, Constants.Build.Path.LegacyArtifactsDirectory));
DeleteDirectory(
executionContext,
description: "legacy staging directory",
path: Path.Combine(existingConfig.BuildDirectory, Constants.Build.Path.LegacyStagingDirectory));
}
// Save any changes to the config file
trackingManager.UpdateTrackingConfig(executionContext, newConfig);
// Prepare the build directory.
// There are 2 ways to provide build directory clean policy.
// 1> set definition variable build.clean or agent.clean.buildDirectory. (on-prem user need to use this, since there is no Web UI in TFS 2016)
// 2> select source clean option in definition repository tab. (VSTS will have this option in definition designer UI)
BuildCleanOption cleanOption = GetBuildDirectoryCleanOption(executionContext, workspace);
CreateDirectory(
executionContext,
description: "build directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.BuildDirectory),
deleteExisting: cleanOption == BuildCleanOption.All);
CreateDirectory(
executionContext,
description: "artifacts directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.ArtifactsDirectory),
deleteExisting: true);
CreateDirectory(
executionContext,
description: "test results directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.TestResultsDirectory),
deleteExisting: true);
CreateDirectory(
executionContext,
description: "binaries directory",
path: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.BuildDirectory, Constants.Build.Path.BinariesDirectory),
deleteExisting: cleanOption == BuildCleanOption.Binary);
var defaultSourceDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), newConfig.SourcesDirectory);
CreateDirectory(
executionContext,
description: "source directory",
path: defaultSourceDirectory,
deleteExisting: cleanOption == BuildCleanOption.Source);
// Set the default clone path for each repository (the Checkout task may override this later)
foreach (var repository in repositories)
{
string repoPath = GetDefaultRepositoryPath(executionContext, repository, newConfig);
if (!string.Equals(repoPath, defaultSourceDirectory, StringComparison.Ordinal))
{
CreateDirectory(
executionContext,
description: "repository source directory",
path: repoPath,
deleteExisting: cleanOption == BuildCleanOption.Source);
}
Trace.Info($"Set repository path for repository {repository.Alias} to '{repoPath}'");
repository.Properties.Set <string>(RepositoryPropertyNames.Path, repoPath);
}
return(newConfig);
}
private async Task StartContainerAsync(IExecutionContext executionContext, ContainerInfo container)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(container, nameof(container));
ArgUtil.NotNullOrEmpty(container.ContainerImage, nameof(container.ContainerImage));
Trace.Info($"Container name: {container.ContainerName}");
Trace.Info($"Container image: {container.ContainerImage}");
Trace.Info($"Container registry: {container.ContainerRegistryEndpoint.ToString()}");
Trace.Info($"Container options: {container.ContainerCreateOptions}");
Trace.Info($"Skip container image pull: {container.SkipContainerImagePull}");
foreach (var port in container.UserPortMappings)
{
Trace.Info($"User provided port: {port.Value}");
}
foreach (var volume in container.UserMountVolumes)
{
Trace.Info($"User provided volume: {volume.Value}");
}
// Login to private docker registry
string registryServer = string.Empty;
if (container.ContainerRegistryEndpoint != Guid.Empty)
{
var registryEndpoint = executionContext.Endpoints.FirstOrDefault(x => x.Type == "dockerregistry" && x.Id == container.ContainerRegistryEndpoint);
ArgUtil.NotNull(registryEndpoint, nameof(registryEndpoint));
string username = string.Empty;
string password = string.Empty;
string registryType = string.Empty;
registryEndpoint.Data?.TryGetValue("registrytype", out registryType);
if (string.Equals(registryType, "ACR", StringComparison.OrdinalIgnoreCase))
{
string loginServer = string.Empty;
registryEndpoint.Authorization?.Parameters?.TryGetValue("loginServer", out loginServer);
if (loginServer != null)
{
loginServer = loginServer.ToLower();
}
registryEndpoint.Authorization?.Parameters?.TryGetValue("serviceprincipalid", out username);
registryEndpoint.Authorization?.Parameters?.TryGetValue("serviceprincipalkey", out password);
registryServer = $"https://{loginServer}";
}
else
{
registryEndpoint.Authorization?.Parameters?.TryGetValue("registry", out registryServer);
registryEndpoint.Authorization?.Parameters?.TryGetValue("username", out username);
registryEndpoint.Authorization?.Parameters?.TryGetValue("password", out password);
}
ArgUtil.NotNullOrEmpty(registryServer, nameof(registryServer));
ArgUtil.NotNullOrEmpty(username, nameof(username));
ArgUtil.NotNullOrEmpty(password, nameof(password));
int loginExitCode = await _dockerManger.DockerLogin(executionContext, registryServer, username, password);
if (loginExitCode != 0)
{
throw new InvalidOperationException($"Docker login fail with exit code {loginExitCode}");
}
}
try
{
if (!container.SkipContainerImagePull)
{
if (!string.IsNullOrEmpty(registryServer) &&
registryServer.IndexOf("index.docker.io", StringComparison.OrdinalIgnoreCase) < 0)
{
var registryServerUri = new Uri(registryServer);
if (!container.ContainerImage.StartsWith(registryServerUri.Authority, StringComparison.OrdinalIgnoreCase))
{
container.ContainerImage = $"{registryServerUri.Authority}/{container.ContainerImage}";
}
}
// Pull down docker image with retry up to 3 times
int retryCount = 0;
int pullExitCode = 0;
while (retryCount < 3)
{
pullExitCode = await _dockerManger.DockerPull(executionContext, container.ContainerImage);
if (pullExitCode == 0)
{
break;
}
else
{
retryCount++;
if (retryCount < 3)
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
executionContext.Warning($"Docker pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
}
if (retryCount == 3 && pullExitCode != 0)
{
throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}");
}
}
// Mount folder into container
#if OS_WINDOWS
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Work), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Work))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools))));
#else
string defaultWorkingDirectory = executionContext.Variables.Get(Constants.Variables.System.DefaultWorkingDirectory);
if (string.IsNullOrEmpty(defaultWorkingDirectory))
{
throw new NotSupportedException(StringUtil.Loc("ContainerJobRequireSystemDefaultWorkDir"));
}
string workingDirectory = Path.GetDirectoryName(defaultWorkingDirectory.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar));
container.MountVolumes.Add(new MountVolume(container.TranslateToHostPath(workingDirectory), workingDirectory));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Temp), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Temp))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tasks), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tasks))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)), true));
// Ensure .taskkey file exist so we can mount it.
string taskKeyFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), ".taskkey");
if (!File.Exists(taskKeyFile))
{
File.WriteAllText(taskKeyFile, string.Empty);
}
container.MountVolumes.Add(new MountVolume(taskKeyFile, container.TranslateToContainerPath(taskKeyFile)));
#endif
if (container.IsJobContainer)
{
// See if this container brings its own Node.js
container.ContainerBringNodePath = await _dockerManger.DockerInspect(context : executionContext,
dockerObject : container.ContainerImage,
options : $"--format=\"{{{{index .Config.Labels \\\"{_nodeJsPathLabel}\\\"}}}}\"");
string node;
if (!string.IsNullOrEmpty(container.ContainerBringNodePath))
{
node = container.ContainerBringNodePath;
}
else
{
node = container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node", "bin", $"node{IOUtil.ExeExtension}"));
}
string sleepCommand = $"\"{node}\" -e \"setInterval(function(){{}}, 24 * 60 * 60 * 1000);\"";
container.ContainerCommand = sleepCommand;
}
container.ContainerId = await _dockerManger.DockerCreate(executionContext, container);
ArgUtil.NotNullOrEmpty(container.ContainerId, nameof(container.ContainerId));
if (container.IsJobContainer)
{
executionContext.Variables.Set(Constants.Variables.Agent.ContainerId, container.ContainerId);
}
// Start container
int startExitCode = await _dockerManger.DockerStart(executionContext, container.ContainerId);
if (startExitCode != 0)
{
throw new InvalidOperationException($"Docker start fail with exit code {startExitCode}");
}
}
finally
{
// Logout for private registry
if (!string.IsNullOrEmpty(registryServer))
{
int logoutExitCode = await _dockerManger.DockerLogout(executionContext, registryServer);
if (logoutExitCode != 0)
{
executionContext.Error($"Docker logout fail with exit code {logoutExitCode}");
}
}
}
try
{
// Make sure container is up and running
var psOutputs = await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --filter status=running --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\"");
if (psOutputs.FirstOrDefault(x => !string.IsNullOrEmpty(x))?.StartsWith(container.ContainerId) != true)
{
// container is not up and running, pull docker log for this container.
await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\"");
int logsExitCode = await _dockerManger.DockerLogs(executionContext, container.ContainerId);
if (logsExitCode != 0)
{
executionContext.Warning($"Docker logs fail with exit code {logsExitCode}");
}
executionContext.Warning($"Docker container {container.ContainerId} is not in running state.");
}
}
catch (Exception ex)
{
// pull container log is best effort.
Trace.Error("Catch exception when check container log and container status.");
Trace.Error(ex);
}
// Get port mappings of running container
if (executionContext.Container == null && !container.IsJobContainer)
{
container.AddPortMappings(await _dockerManger.DockerPort(executionContext, container.ContainerId));
foreach (var port in container.PortMappings)
{
executionContext.Variables.Set(
$"{Constants.Variables.Agent.ServicePortPrefix}.{container.ContainerNetworkAlias}.ports.{port.ContainerPort}",
$"{port.HostPort}");
}
}
#if !OS_WINDOWS
if (container.IsJobContainer)
{
// Ensure bash exist in the image
int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"sh -c \"command -v bash\"");
if (execWhichBashExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execWhichBashExitCode}");
}
// Get current username
container.CurrentUserName = (await ExecuteCommandAsync(executionContext, "whoami", string.Empty)).FirstOrDefault();
ArgUtil.NotNullOrEmpty(container.CurrentUserName, nameof(container.CurrentUserName));
// Get current userId
container.CurrentUserId = (await ExecuteCommandAsync(executionContext, "id", $"-u {container.CurrentUserName}")).FirstOrDefault();
ArgUtil.NotNullOrEmpty(container.CurrentUserId, nameof(container.CurrentUserId));
executionContext.Output(StringUtil.Loc("CreateUserWithSameUIDInsideContainer", container.CurrentUserId));
// Create an user with same uid as the agent run as user inside the container.
// All command execute in docker will run as Root by default,
// this will cause the agent on the host machine doesn't have permission to any new file/folder created inside the container.
// So, we create a user account with same UID inside the container and let all docker exec command run as that user.
string containerUserName = string.Empty;
// We need to find out whether there is a user with same UID inside the container
List <string> userNames = new List <string>();
int execGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"grep {container.CurrentUserId} /etc/passwd | cut -f1 -d:\"", userNames);
if (execGrepExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGrepExitCode}");
}
if (userNames.Count > 0)
{
// check all potential username that might match the UID.
foreach (string username in userNames)
{
int execIdExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"id -u {username}");
if (execIdExitCode == 0)
{
containerUserName = username;
break;
}
}
}
// Create a new user with same UID
if (string.IsNullOrEmpty(containerUserName))
{
containerUserName = $"{container.CurrentUserName}_azpcontainer";
int execUseraddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"useradd -m -u {container.CurrentUserId} {containerUserName}");
if (execUseraddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execUseraddExitCode}");
}
}
executionContext.Output(StringUtil.Loc("GrantContainerUserSUDOPrivilege", containerUserName));
// Create a new group for giving sudo permission
int execGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd azure_pipelines_sudo");
if (execGroupaddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGroupaddExitCode}");
}
// Add the new created user to the new created sudo group.
int execUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G azure_pipelines_sudo {containerUserName}");
if (execUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execUsermodExitCode}");
}
// Allow the new sudo group run any sudo command without providing password.
int execEchoExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"su -c \"echo '%azure_pipelines_sudo ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers\"");
if (execUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execEchoExitCode}");
}
bool setupDockerGroup = executionContext.Variables.GetBoolean("VSTS_SETUP_DOCKERGROUP") ?? StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("VSTS_SETUP_DOCKERGROUP"), true);
if (setupDockerGroup)
{
executionContext.Output(StringUtil.Loc("AllowContainerUserRunDocker", containerUserName));
// Get docker.sock group id on Host
string dockerSockGroupId = (await ExecuteCommandAsync(executionContext, "stat", $"-c %g /var/run/docker.sock")).FirstOrDefault();
// We need to find out whether there is a group with same GID inside the container
string existingGroupName = null;
List <string> groupsOutput = new List <string>();
int execGroupGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"cat /etc/group\"", groupsOutput);
if (execGroupGrepExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGroupGrepExitCode}");
}
if (groupsOutput.Count > 0)
{
// check all potential groups that might match the GID.
foreach (string groupOutput in groupsOutput)
{
if (!string.IsNullOrEmpty(groupOutput))
{
var groupSegments = groupOutput.Split(':');
if (groupSegments.Length != 4)
{
Trace.Warning($"Unexpected output from /etc/group: '{groupOutput}'");
}
else
{
// the output of /etc/group should looks like `group:x:gid:`
var groupName = groupSegments[0];
var groupId = groupSegments[2];
if (string.Equals(dockerSockGroupId, groupId))
{
existingGroupName = groupName;
break;
}
}
}
}
}
if (string.IsNullOrEmpty(existingGroupName))
{
// create a new group with same gid
existingGroupName = "azure_pipelines_docker";
int execDockerGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd -g {dockerSockGroupId} azure_pipelines_docker");
if (execDockerGroupaddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execDockerGroupaddExitCode}");
}
}
// Add the new created user to the docker socket group.
int execGroupUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G {existingGroupName} {containerUserName}");
if (execGroupUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGroupUsermodExitCode}");
}
}
}
#endif
}
protected override async Task ProcessCommandInternalAsync(
AgentTaskPluginExecutionContext context,
string targetPath,
string artifactName,
CancellationToken token)
{
// Create target directory if absent
string fullPath = Path.GetFullPath(targetPath);
bool isDir = Directory.Exists(fullPath);
if (!isDir)
{
Directory.CreateDirectory(fullPath);
}
// Project ID
// TODO: use a constant for project id, which is currently defined in Microsoft.VisualStudio.Services.Agent.Constants.Variables.System.TeamProjectId (Ting)
string guidStr = context.Variables.GetValueOrDefault("system.teamProjectId")?.Value;
Guid.TryParse(guidStr, out Guid projectId);
ArgUtil.NotEmpty(projectId, nameof(projectId));
// Build ID
int buildId = 0;
string buildIdStr = context.GetInput(ArtifactEventProperties.PipelineId, required: false);
// Determine the build id
if (Int32.TryParse(buildIdStr, out buildId) && buildId != 0)
{
// A) Build Id provided by user input
context.Output(StringUtil.Loc("DownloadingFromBuild", buildId));
}
else
{
// B) Build Id provided by environment
buildIdStr = context.Variables.GetValueOrDefault(BuildVariables.BuildId)?.Value ?? string.Empty;
if (int.TryParse(buildIdStr, out buildId) && buildId != 0)
{
context.Output(StringUtil.Loc("DownloadingFromBuild", buildId));
}
else
{
string hostType = context.Variables.GetValueOrDefault("system.hosttype")?.Value;
if (string.Equals(hostType, "Release", StringComparison.OrdinalIgnoreCase) ||
string.Equals(hostType, "DeploymentGroup", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException(StringUtil.Loc("BuildIdIsNotAvailable", hostType ?? string.Empty));
}
else if (!string.Equals(hostType, "Build", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException(StringUtil.Loc("CannotDownloadFromCurrentEnvironment", hostType ?? string.Empty));
}
else
{
// This should not happen since the build id comes from build environment. But a user may override that so we must be careful.
throw new ArgumentException(StringUtil.Loc("BuildIdIsNotValid", buildIdStr));
}
}
}
// Download from VSTS BlobStore
context.Output(StringUtil.Loc("DownloadArtifactTo", targetPath));
PipelineArtifactServer server = new PipelineArtifactServer();
await server.DownloadAsync(context, projectId, buildId, artifactName, targetPath, token);
context.Output(StringUtil.Loc("DownloadArtifactFinished"));
}
public async Task GetAsync(string localPath, bool quiet = false)
{
ArgUtil.NotNullOrEmpty(localPath, nameof(localPath));
await RunCommandAsync(FormatFlags.OmitCollectionUrl, quiet, 3, "vc", "get", $"/version:{SourceVersion}", "/recursive", "/overwrite", localPath);
}
public void SetupProxy(string proxyUrl, string proxyUsername, string proxyPassword)
{
ArgUtil.File(AppConfigFile, "tf.exe.config");
if (!File.Exists(AppConfigRestoreFile))
{
ExecutionContext.Debug("Take snapshot of current appconfig for restore modified appconfig.");
File.Copy(AppConfigFile, AppConfigRestoreFile);
}
else
{
// cleanup any appconfig changes from previous build.
CleanupProxySetting();
}
if (!string.IsNullOrEmpty(proxyUrl))
{
XmlDocument appConfig = new XmlDocument();
using (var appConfigStream = new FileStream(AppConfigFile, FileMode.Open, FileAccess.Read))
{
appConfig.Load(appConfigStream);
}
var configuration = appConfig.SelectSingleNode("configuration");
ArgUtil.NotNull(configuration, "configuration");
var exist_defaultProxy = appConfig.SelectSingleNode("configuration/system.net/defaultProxy");
if (exist_defaultProxy == null)
{
var system_net = appConfig.SelectSingleNode("configuration/system.net");
if (system_net == null)
{
ExecutionContext.Debug("Create system.net section in appconfg.");
system_net = appConfig.CreateElement("system.net");
}
ExecutionContext.Debug("Create defaultProxy section in appconfg.");
var defaultProxy = appConfig.CreateElement("defaultProxy");
defaultProxy.SetAttribute("useDefaultCredentials", "True");
ExecutionContext.Debug("Create proxy section in appconfg.");
var proxy = appConfig.CreateElement("proxy");
proxy.SetAttribute("proxyaddress", proxyUrl);
defaultProxy.AppendChild(proxy);
system_net.AppendChild(defaultProxy);
configuration.AppendChild(system_net);
using (var appConfigStream = new FileStream(AppConfigFile, FileMode.Open, FileAccess.ReadWrite))
{
appConfig.Save(appConfigStream);
}
}
else
{
//proxy setting exist.
ExecutionContext.Debug("Proxy setting already exist in app.config file.");
}
// when tf.exe talk to any devfabric site, it will always bypass proxy.
// for testing, we need set this variable to let tf.exe hit the proxy server on devfabric.
if (Endpoint.Url.Host.Contains(".me.tfsallin.net") || Endpoint.Url.Host.Contains(".vsts.me"))
{
ExecutionContext.Debug("Set TFS_BYPASS_PROXY_ON_LOCAL on devfabric.");
AdditionalEnvironmentVariables["TFS_BYPASS_PROXY_ON_LOCAL"] = "0";
}
}
}
public override async Task WorkspacesRemoveAsync(ITfsVCWorkspace workspace)
{
ArgUtil.NotNull(workspace, nameof(workspace));
await RunCommandAsync("vc", "workspace", $"/remove:{workspace.Name};{workspace.Owner}");
}
// TODO: Remove AddAsync after last-saved-checkin-metadata problem is fixed properly.
public async Task AddAsync(string localPath)
{
ArgUtil.NotNullOrEmpty(localPath, nameof(localPath));
await RunPorcelainCommandAsync(FormatFlags.OmitCollectionUrl, "vc", "add", localPath);
}
public async Task WorkspaceDeleteAsync(ITfsVCWorkspace workspace)
{
ArgUtil.NotNull(workspace, nameof(workspace));
await RunCommandAsync("vc", "workspace", "/delete", $"{workspace.Name};{workspace.Owner}");
}
public async Task WorkfoldUnmapAsync(string serverPath)
{
ArgUtil.NotNullOrEmpty(serverPath, nameof(serverPath));
await RunCommandAsync("vc", "workfold", "/unmap", $"/workspace:{WorkspaceName}", serverPath);
}
public async Task WorkfoldMapAsync(string serverPath, string localPath)
{
ArgUtil.NotNullOrEmpty(serverPath, nameof(serverPath));
ArgUtil.NotNullOrEmpty(localPath, nameof(localPath));
await RunCommandAsync(3, "vc", "workfold", "/map", $"/workspace:{WorkspaceName}", serverPath, localPath);
}
public async Task GetSourceAsync(
IExecutionContext executionContext,
ServiceEndpoint endpoint,
CancellationToken cancellationToken)
{
Trace.Entering();
// Validate args.
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(endpoint, nameof(endpoint));
ISvnCommandManager svn = HostContext.CreateService <ISvnCommandManager>();
svn.Init(executionContext, endpoint, cancellationToken);
// Determine the sources directory.
string sourcesDirectory = GetEndpointData(endpoint, Constants.EndpointData.SourcesDirectory);
executionContext.Debug($"sourcesDirectory={sourcesDirectory}");
ArgUtil.NotNullOrEmpty(sourcesDirectory, nameof(sourcesDirectory));
string sourceBranch = GetEndpointData(endpoint, Constants.EndpointData.SourceBranch);
executionContext.Debug($"sourceBranch={sourceBranch}");
string revision = GetEndpointData(endpoint, Constants.EndpointData.SourceVersion);
if (string.IsNullOrWhiteSpace(revision))
{
revision = "HEAD";
}
executionContext.Debug($"revision={revision}");
bool clean = endpoint.Data.ContainsKey(EndpointData.Clean) &&
StringUtil.ConvertToBoolean(endpoint.Data[EndpointData.Clean], defaultValue: false);
executionContext.Debug($"clean={clean}");
// Get the definition mappings.
List <SvnMappingDetails> allMappings = JsonConvert.DeserializeObject <SvnWorkspace>
(endpoint.Data[EndpointData.SvnWorkspaceMapping]).Mappings;
if (executionContext.Variables.System_Debug.HasValue && executionContext.Variables.System_Debug.Value)
{
allMappings.ForEach(m => executionContext.Debug($"ServerPath: {m.ServerPath}, LocalPath: {m.LocalPath}, Depth: {m.Depth}, Revision: {m.Revision}, IgnoreExternals: {m.IgnoreExternals}"));
}
Dictionary <string, SvnMappingDetails> normalizedMappings = svn.NormalizeMappings(allMappings);
if (executionContext.Variables.System_Debug.HasValue && executionContext.Variables.System_Debug.Value)
{
executionContext.Debug($"Normalized mappings count: {normalizedMappings.Count}");
normalizedMappings.ToList().ForEach(p => executionContext.Debug($" [{p.Key}] ServerPath: {p.Value.ServerPath}, LocalPath: {p.Value.LocalPath}, Depth: {p.Value.Depth}, Revision: {p.Value.Revision}, IgnoreExternals: {p.Value.IgnoreExternals}"));
}
string normalizedBranch = svn.NormalizeRelativePath(sourceBranch, '/', '\\');
executionContext.Output(StringUtil.Loc("SvnSyncingRepo", endpoint.Name));
string effectiveRevision = await svn.UpdateWorkspace(
sourcesDirectory,
normalizedMappings,
clean,
normalizedBranch,
revision);
executionContext.Output(StringUtil.Loc("SvnBranchCheckedOut", normalizedBranch, endpoint.Name, effectiveRevision));
Trace.Verbose("Leaving SvnSourceProvider.GetSourceAsync");
}
public async Task RunAsync()
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(Data, nameof(Data));
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
ArgUtil.NotNull(Inputs, nameof(Inputs));
ArgUtil.Directory(TaskDirectory, nameof(TaskDirectory));
// Resolve the target script.
ArgUtil.NotNullOrEmpty(Data.Target, nameof(Data.Target));
string scriptFile = Path.Combine(TaskDirectory, Data.Target);
ArgUtil.File(scriptFile, nameof(scriptFile));
// Determine the working directory.
string workingDirectory = Data.WorkingDirectory;
if (String.IsNullOrEmpty(workingDirectory))
{
workingDirectory = Path.GetDirectoryName(scriptFile);
}
else
{
if (!Directory.Exists(workingDirectory))
{
Directory.CreateDirectory(workingDirectory);
}
}
// scriptName
AddEnvironmentVariable("VSTSPSHOSTSCRIPTNAME", scriptFile);
// workingFolder
AddEnvironmentVariable("VSTSPSHOSTWORKINGFOLDER", workingDirectory);
// outputPreference
AddEnvironmentVariable("VSTSPSHOSTOUTPUTPREFER", ExecutionContext.WriteDebug ? "Continue" : "SilentlyContinue");
// inputParameters
if (Inputs.Count > 0)
{
AddEnvironmentVariable("VSTSPSHOSTINPUTPARAMETER", JsonUtility.ToString(Inputs));
}
List <String> arguments = new List <string>();
Dictionary <String, String> argumentParameters = new Dictionary <String, String>();
if (string.IsNullOrEmpty(Data.ArgumentFormat))
{
// treatInputsAsArguments
AddEnvironmentVariable("VSTSPSHOSTINPUTISARG", "True");
}
else
{
MatchCollection matches = _argumentMatching.Matches(Data.ArgumentFormat);
if (matches[0].Value.StartsWith("-"))
{
String currentKey = String.Empty;
foreach (Match match in matches)
{
if (match.Value.StartsWith("-"))
{
currentKey = match.Value.Trim('-');
argumentParameters.Add(currentKey, String.Empty);
}
else if (!match.Value.StartsWith("-") && !String.IsNullOrEmpty(currentKey))
{
argumentParameters[currentKey] = match.Value;
currentKey = String.Empty;
}
else
{
throw new Exception($"Found value {match.Value} with no corresponding named parameter");
}
}
}
else
{
foreach (Match match in matches)
{
arguments.Add(match.Value);
}
}
// arguments
if (arguments.Count > 0)
{
AddEnvironmentVariable("VSTSPSHOSTARGS", JsonUtility.ToString(arguments));
}
// argumentParameters
if (argumentParameters.Count > 0)
{
AddEnvironmentVariable("VSTSPSHOSTARGPARAMETER", JsonUtility.ToString(argumentParameters));
}
}
// additionalStatement
List <Tuple <String, List <Tuple <String, String> > > > additionalStatement = GetAdditionalCommandsForAzurePowerShell(Inputs);
if (additionalStatement.Count > 0)
{
AddEnvironmentVariable("VSTSPSHOSTSTATEMENTS", JsonUtility.ToString(additionalStatement));
}
// push all variable.
foreach (var variable in ExecutionContext.Variables.Public.Concat(ExecutionContext.Variables.Private))
{
AddEnvironmentVariable("VSTSPSHOSTVAR_" + variable.Key, variable.Value);
}
// push all public variable.
foreach (var variable in ExecutionContext.Variables.Public)
{
AddEnvironmentVariable("VSTSPSHOSTPUBVAR_" + variable.Key, variable.Value);
}
// push all endpoints
List <String> ids = new List <string>();
foreach (ServiceEndpoint endpoint in ExecutionContext.Endpoints)
{
string partialKey = null;
if (string.Equals(endpoint.Name, ServiceEndpoints.SystemVssConnection, StringComparison.OrdinalIgnoreCase))
{
partialKey = ServiceEndpoints.SystemVssConnection.ToUpperInvariant();
AddEnvironmentVariable("VSTSPSHOSTSYSTEMENDPOINT_URL", endpoint.Url.ToString());
AddEnvironmentVariable("VSTSPSHOSTSYSTEMENDPOINT_AUTH", JsonUtility.ToString(endpoint.Authorization));
}
else
{
partialKey = endpoint.Id.ToString("D").ToUpperInvariant();
ids.Add(partialKey);
AddEnvironmentVariable("VSTSPSHOSTENDPOINT_URL_" + partialKey, endpoint.Url.ToString());
AddEnvironmentVariable("VSTSPSHOSTENDPOINT_NAME_" + partialKey, endpoint.Name);
AddEnvironmentVariable("VSTSPSHOSTENDPOINT_TYPE_" + partialKey, endpoint.Type);
AddEnvironmentVariable("VSTSPSHOSTENDPOINT_AUTH_" + partialKey, JsonUtility.ToString(endpoint.Authorization));
AddEnvironmentVariable("VSTSPSHOSTENDPOINT_DATA_" + partialKey, JsonUtility.ToString(endpoint.Data));
}
}
if (ids.Count > 0)
{
AddEnvironmentVariable("VSTSPSHOSTENDPOINT_IDS", JsonUtility.ToString(ids));
}
// Invoke the process.
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += OnDataReceived;
processInvoker.ErrorDataReceived += OnDataReceived;
try
{
String vstsPSHostExe = Path.Combine(IOUtil.GetExternalsPath(), "vstshost", "LegacyVSTSPowerShellHost.exe");
Int32 exitCode = await processInvoker.ExecuteAsync(workingDirectory : workingDirectory,
fileName : vstsPSHostExe,
arguments : "",
environment : Environment,
cancellationToken : ExecutionContext.CancellationToken);
// the exit code from vstsPSHost.exe indicate how many error record we get during execution
// -1 exit code means infrastructure failure of Host itself.
// this is to match current handler's logic.
if (exitCode > 0)
{
if (ExecutionContext.Result != null)
{
Trace.Info($"Task result already set. Not failing due to error count ({exitCode}).");
}
else
{
// We fail task and add issue.
ExecutionContext.Result = TaskResult.Failed;
ExecutionContext.Error(StringUtil.Loc("PSScriptError", exitCode));
}
}
else if (exitCode < 0)
{
// We fail task and add issue.
ExecutionContext.Result = TaskResult.Failed;
ExecutionContext.Error(StringUtil.Loc("VSTSHostNonZeroReturn", exitCode));
}
}
finally
{
processInvoker.OutputDataReceived -= OnDataReceived;
processInvoker.ErrorDataReceived -= OnDataReceived;
}
}
}
public static int Main(string[] args)
{
Console.CancelKeyPress += Console_CancelKeyPress;
try
{
ArgUtil.NotNull(args, nameof(args));
ArgUtil.Equal(2, args.Length, nameof(args.Length));
string pluginType = args[0];
if (string.Equals("task", pluginType, StringComparison.OrdinalIgnoreCase))
{
string assemblyQualifiedName = args[1];
ArgUtil.NotNullOrEmpty(assemblyQualifiedName, nameof(assemblyQualifiedName));
// Set encoding to UTF8, process invoker will use UTF8 write to STDIN
Console.InputEncoding = Encoding.UTF8;
string serializedContext = Console.ReadLine();
ArgUtil.NotNullOrEmpty(serializedContext, nameof(serializedContext));
AgentTaskPluginExecutionContext executionContext = StringUtil.ConvertFromJson <AgentTaskPluginExecutionContext>(serializedContext);
ArgUtil.NotNull(executionContext, nameof(executionContext));
AssemblyLoadContext.Default.Resolving += ResolveAssembly;
try
{
Type type = Type.GetType(assemblyQualifiedName, throwOnError: true);
var taskPlugin = Activator.CreateInstance(type) as IAgentTaskPlugin;
ArgUtil.NotNull(taskPlugin, nameof(taskPlugin));
taskPlugin.RunAsync(executionContext, tokenSource.Token).GetAwaiter().GetResult();
}
catch (Exception ex)
{
// any exception throw from plugin will fail the task.
executionContext.Error(ex.ToString());
}
finally
{
AssemblyLoadContext.Default.Resolving -= ResolveAssembly;
}
return(0);
}
else if (string.Equals("command", pluginType, StringComparison.OrdinalIgnoreCase))
{
string assemblyQualifiedName = args[1];
ArgUtil.NotNullOrEmpty(assemblyQualifiedName, nameof(assemblyQualifiedName));
string serializedContext = Console.ReadLine();
ArgUtil.NotNullOrEmpty(serializedContext, nameof(serializedContext));
AgentCommandPluginExecutionContext executionContext = StringUtil.ConvertFromJson <AgentCommandPluginExecutionContext>(serializedContext);
ArgUtil.NotNull(executionContext, nameof(executionContext));
AssemblyLoadContext.Default.Resolving += ResolveAssembly;
try
{
Type type = Type.GetType(assemblyQualifiedName, throwOnError: true);
var commandPlugin = Activator.CreateInstance(type) as IAgentCommandPlugin;
ArgUtil.NotNull(commandPlugin, nameof(commandPlugin));
commandPlugin.ProcessCommandAsync(executionContext, tokenSource.Token).GetAwaiter().GetResult();
}
catch (Exception ex)
{
// any exception throw from plugin will fail the command.
executionContext.Error(ex.ToString());
}
finally
{
AssemblyLoadContext.Default.Resolving -= ResolveAssembly;
}
return(0);
}
else
{
throw new ArgumentOutOfRangeException(pluginType);
}
}
catch (Exception ex)
{
// infrastructure failure.
Console.Error.WriteLine(ex.ToString());
return(1);
}
finally
{
Console.CancelKeyPress -= Console_CancelKeyPress;
}
}
public Definition LoadAction(IExecutionContext executionContext, Pipelines.ActionStep action)
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(action, nameof(action));
// Initialize the definition wrapper object.
var definition = new Definition()
{
Data = new ActionDefinitionData()
};
if (action.Reference.Type == Pipelines.ActionSourceType.ContainerRegistry)
{
Trace.Info("Load action that reference container from registry.");
CachedActionContainers.TryGetValue(action.Id, out var container);
ArgUtil.NotNull(container, nameof(container));
definition.Data.Execution = new ContainerActionExecutionData()
{
Image = container.ContainerImage
};
Trace.Info($"Using action container image: {container.ContainerImage}.");
}
else if (action.Reference.Type == Pipelines.ActionSourceType.Repository)
{
string actionDirectory = null;
var repoAction = action.Reference as Pipelines.RepositoryPathReference;
if (string.Equals(repoAction.RepositoryType, Pipelines.PipelineConstants.SelfAlias, StringComparison.OrdinalIgnoreCase))
{
actionDirectory = executionContext.GetGitHubContext("workspace");
if (!string.IsNullOrEmpty(repoAction.Path))
{
actionDirectory = Path.Combine(actionDirectory, repoAction.Path);
}
}
else
{
actionDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), repoAction.Name.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), repoAction.Ref);
if (!string.IsNullOrEmpty(repoAction.Path))
{
actionDirectory = Path.Combine(actionDirectory, repoAction.Path);
}
}
Trace.Info($"Load action that reference repository from '{actionDirectory}'");
definition.Directory = actionDirectory;
string manifestFile = Path.Combine(actionDirectory, Constants.Path.ActionManifestYmlFile);
string manifestFileYaml = Path.Combine(actionDirectory, Constants.Path.ActionManifestYamlFile);
string dockerFile = Path.Combine(actionDirectory, "Dockerfile");
string dockerFileLowerCase = Path.Combine(actionDirectory, "dockerfile");
if (File.Exists(manifestFile) || File.Exists(manifestFileYaml))
{
var manifestManager = HostContext.GetService <IActionManifestManager>();
if (File.Exists(manifestFile))
{
definition.Data = manifestManager.Load(executionContext, manifestFile);
}
else
{
definition.Data = manifestManager.Load(executionContext, manifestFileYaml);
}
Trace.Verbose($"Action friendly name: '{definition.Data.Name}'");
Trace.Verbose($"Action description: '{definition.Data.Description}'");
if (definition.Data.Inputs != null)
{
foreach (var input in definition.Data.Inputs)
{
Trace.Verbose($"Action input: '{input.Key.ToString()}' default to '{input.Value.ToString()}'");
}
}
if (definition.Data.Execution.ExecutionType == ActionExecutionType.Container)
{
var containerAction = definition.Data.Execution as ContainerActionExecutionData;
Trace.Info($"Action container Dockerfile/image: {containerAction.Image}.");
if (containerAction.Arguments != null)
{
Trace.Info($"Action container args: {StringUtil.ConvertToJson(containerAction.Arguments)}.");
}
if (containerAction.Environment != null)
{
Trace.Info($"Action container env: {StringUtil.ConvertToJson(containerAction.Environment)}.");
}
if (!string.IsNullOrEmpty(containerAction.EntryPoint))
{
Trace.Info($"Action container entrypoint: {containerAction.EntryPoint}.");
}
if (!string.IsNullOrEmpty(containerAction.Cleanup))
{
Trace.Info($"Action container cleanup entrypoint: {containerAction.Cleanup}.");
}
if (CachedActionContainers.TryGetValue(action.Id, out var container))
{
Trace.Info($"Image '{containerAction.Image}' already built/pulled, use image: {container.ContainerImage}.");
containerAction.Image = container.ContainerImage;
}
}
else if (definition.Data.Execution.ExecutionType == ActionExecutionType.NodeJS)
{
var nodeAction = definition.Data.Execution as NodeJSActionExecutionData;
Trace.Info($"Action node.js file: {nodeAction.Script}.");
Trace.Info($"Action cleanup node.js file: {nodeAction.Cleanup ?? "N/A"}.");
}
else if (definition.Data.Execution.ExecutionType == ActionExecutionType.Plugin)
{
var pluginAction = definition.Data.Execution as PluginActionExecutionData;
var pluginManager = HostContext.GetService <IRunnerPluginManager>();
var plugin = pluginManager.GetPluginAction(pluginAction.Plugin);
ArgUtil.NotNull(plugin, pluginAction.Plugin);
ArgUtil.NotNullOrEmpty(plugin.PluginTypeName, pluginAction.Plugin);
pluginAction.Plugin = plugin.PluginTypeName;
Trace.Info($"Action plugin: {plugin.PluginTypeName}.");
if (!string.IsNullOrEmpty(plugin.PostPluginTypeName))
{
pluginAction.Cleanup = plugin.PostPluginTypeName;
Trace.Info($"Action cleanup plugin: {plugin.PluginTypeName}.");
}
}
else
{
throw new NotSupportedException(definition.Data.Execution.ExecutionType.ToString());
}
}
else if (File.Exists(dockerFile))
{
if (CachedActionContainers.TryGetValue(action.Id, out var container))
{
definition.Data.Execution = new ContainerActionExecutionData()
{
Image = container.ContainerImage
};
}
else
{
definition.Data.Execution = new ContainerActionExecutionData()
{
Image = dockerFile
};
}
}
else if (File.Exists(dockerFileLowerCase))
{
if (CachedActionContainers.TryGetValue(action.Id, out var container))
{
definition.Data.Execution = new ContainerActionExecutionData()
{
Image = container.ContainerImage
};
}
else
{
definition.Data.Execution = new ContainerActionExecutionData()
{
Image = dockerFileLowerCase
};
}
}
else
{
var fullPath = IOUtil.ResolvePath(actionDirectory, "."); // resolve full path without access filesystem.
throw new NotSupportedException($"Can't find 'action.yml', 'action.yaml' or 'Dockerfile' under '{fullPath}'. Did you forget to run actions/checkout before running your local action?");
}
}
else if (action.Reference.Type == Pipelines.ActionSourceType.Script)
{
definition.Data.Execution = new ScriptActionExecutionData();
definition.Data.Name = "Run";
definition.Data.Description = "Execute a script";
}
else
{
throw new NotSupportedException(action.Reference.Type.ToString());
}
return(definition);
}
public async Task StartContainersAsync(IExecutionContext executionContext, object data)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
List <ContainerInfo> containers = data as List <ContainerInfo>;
ArgUtil.NotNull(containers, nameof(containers));
// Check whether we are inside a container.
// Our container feature requires to map working directory from host to the container.
// If we are already inside a container, we will not able to find out the real working direcotry path on the host.
#if OS_WINDOWS
// service CExecSvc is Container Execution Agent.
ServiceController[] scServices = ServiceController.GetServices();
if (scServices.Any(x => String.Equals(x.ServiceName, "cexecsvc", StringComparison.OrdinalIgnoreCase) && x.Status == ServiceControllerStatus.Running))
{
throw new NotSupportedException(StringUtil.Loc("AgentAlreadyInsideContainer"));
}
#elif OS_RHEL6
// Red Hat and CentOS 6 do not support the container feature
throw new NotSupportedException(StringUtil.Loc("AgentDoesNotSupportContainerFeatureRhel6"));
#else
try
{
var initProcessCgroup = File.ReadLines("/proc/1/cgroup");
if (initProcessCgroup.Any(x => x.IndexOf(":/docker/", StringComparison.OrdinalIgnoreCase) >= 0))
{
throw new NotSupportedException(StringUtil.Loc("AgentAlreadyInsideContainer"));
}
}
catch (Exception ex) when(ex is FileNotFoundException || ex is DirectoryNotFoundException)
{
// if /proc/1/cgroup doesn't exist, we are not inside a container
}
#endif
#if OS_WINDOWS
// Check OS version (Windows server 1803 is required)
object windowsInstallationType = Registry.GetValue(@"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion", "InstallationType", defaultValue: null);
ArgUtil.NotNull(windowsInstallationType, nameof(windowsInstallationType));
object windowsReleaseId = Registry.GetValue(@"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion", "ReleaseId", defaultValue: null);
ArgUtil.NotNull(windowsReleaseId, nameof(windowsReleaseId));
executionContext.Debug($"Current Windows version: '{windowsReleaseId} ({windowsInstallationType})'");
if (int.TryParse(windowsReleaseId.ToString(), out int releaseId))
{
if (!windowsInstallationType.ToString().StartsWith("Server", StringComparison.OrdinalIgnoreCase) || releaseId < 1803)
{
throw new NotSupportedException(StringUtil.Loc("ContainerWindowsVersionRequirement"));
}
}
else
{
throw new ArgumentOutOfRangeException(@"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\ReleaseId");
}
#endif
// Check docker client/server version
DockerVersion dockerVersion = await _dockerManger.DockerVersion(executionContext);
ArgUtil.NotNull(dockerVersion.ServerVersion, nameof(dockerVersion.ServerVersion));
ArgUtil.NotNull(dockerVersion.ClientVersion, nameof(dockerVersion.ClientVersion));
#if OS_WINDOWS
Version requiredDockerEngineAPIVersion = new Version(1, 30); // Docker-EE version 17.6
#else
Version requiredDockerEngineAPIVersion = new Version(1, 35); // Docker-CE version 17.12
#endif
if (dockerVersion.ServerVersion < requiredDockerEngineAPIVersion)
{
throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerServerVersion", requiredDockerEngineAPIVersion, _dockerManger.DockerPath, dockerVersion.ServerVersion));
}
if (dockerVersion.ClientVersion < requiredDockerEngineAPIVersion)
{
throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerClientVersion", requiredDockerEngineAPIVersion, _dockerManger.DockerPath, dockerVersion.ClientVersion));
}
// Clean up containers left by previous runs
executionContext.Debug($"Delete stale containers from previous jobs");
var staleContainers = await _dockerManger.DockerPS(executionContext, $"--all --quiet --no-trunc --filter \"label={_dockerManger.DockerInstanceLabel}\"");
foreach (var staleContainer in staleContainers)
{
int containerRemoveExitCode = await _dockerManger.DockerRemove(executionContext, staleContainer);
if (containerRemoveExitCode != 0)
{
executionContext.Warning($"Delete stale containers failed, docker rm fail with exit code {containerRemoveExitCode} for container {staleContainer}");
}
}
executionContext.Debug($"Delete stale container networks from previous jobs");
int networkPruneExitCode = await _dockerManger.DockerNetworkPrune(executionContext);
if (networkPruneExitCode != 0)
{
executionContext.Warning($"Delete stale container networks failed, docker network prune fail with exit code {networkPruneExitCode}");
}
// Create local docker network for this job to avoid port conflict when multiple agents run on same machine.
// All containers within a job join the same network
await CreateContainerNetworkAsync(executionContext, _containerNetwork);
containers.ForEach(container => container.ContainerNetwork = _containerNetwork);
foreach (var container in containers)
{
await StartContainerAsync(executionContext, container);
}
foreach (var container in containers.Where(c => !c.IsJobContainer))
{
await ContainerHealthcheck(executionContext, container);
}
}
public async Task <List <JobExtensionRunner> > PrepareActionsAsync(IExecutionContext executionContext, IEnumerable <Pipelines.JobStep> steps)
{
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(steps, nameof(steps));
executionContext.Output("Prepare all required actions");
Dictionary <string, List <Guid> > imagesToPull = new Dictionary <string, List <Guid> >(StringComparer.OrdinalIgnoreCase);
Dictionary <string, List <Guid> > imagesToBuild = new Dictionary <string, List <Guid> >(StringComparer.OrdinalIgnoreCase);
Dictionary <string, ActionContainer> imagesToBuildInfo = new Dictionary <string, ActionContainer>(StringComparer.OrdinalIgnoreCase);
List <JobExtensionRunner> containerSetupSteps = new List <JobExtensionRunner>();
IEnumerable <Pipelines.ActionStep> actions = steps.OfType <Pipelines.ActionStep>();
// TODO: Deprecate the PREVIEW_ACTION_TOKEN
// Log even if we aren't using it to ensure users know.
if (!string.IsNullOrEmpty(executionContext.Variables.Get("PREVIEW_ACTION_TOKEN")))
{
executionContext.Warning("The 'PREVIEW_ACTION_TOKEN' secret is deprecated. Please remove it from the repository's secrets");
}
// Clear the cache (for self-hosted runners)
// Note, temporarily avoid this step for the on-premises product, to avoid rate limiting.
var configurationStore = HostContext.GetService <IConfigurationStore>();
var isHostedServer = configurationStore.GetSettings().IsHostedServer;
if (isHostedServer)
{
IOUtil.DeleteDirectory(HostContext.GetDirectory(WellKnownDirectory.Actions), executionContext.CancellationToken);
}
foreach (var action in actions)
{
if (action.Reference.Type == Pipelines.ActionSourceType.ContainerRegistry)
{
ArgUtil.NotNull(action, nameof(action));
var containerReference = action.Reference as Pipelines.ContainerRegistryReference;
ArgUtil.NotNull(containerReference, nameof(containerReference));
ArgUtil.NotNullOrEmpty(containerReference.Image, nameof(containerReference.Image));
if (!imagesToPull.ContainsKey(containerReference.Image))
{
imagesToPull[containerReference.Image] = new List <Guid>();
}
Trace.Info($"Action {action.Name} ({action.Id}) needs to pull image '{containerReference.Image}'");
imagesToPull[containerReference.Image].Add(action.Id);
}
else if (action.Reference.Type == Pipelines.ActionSourceType.Repository)
{
// only download the repository archive
await DownloadRepositoryActionAsync(executionContext, action);
// more preparation base on content in the repository (action.yml)
var setupInfo = PrepareRepositoryActionAsync(executionContext, action);
if (setupInfo != null)
{
if (!string.IsNullOrEmpty(setupInfo.Image))
{
if (!imagesToPull.ContainsKey(setupInfo.Image))
{
imagesToPull[setupInfo.Image] = new List <Guid>();
}
Trace.Info($"Action {action.Name} ({action.Id}) from repository '{setupInfo.ActionRepository}' needs to pull image '{setupInfo.Image}'");
imagesToPull[setupInfo.Image].Add(action.Id);
}
else
{
ArgUtil.NotNullOrEmpty(setupInfo.ActionRepository, nameof(setupInfo.ActionRepository));
if (!imagesToBuild.ContainsKey(setupInfo.ActionRepository))
{
imagesToBuild[setupInfo.ActionRepository] = new List <Guid>();
}
Trace.Info($"Action {action.Name} ({action.Id}) from repository '{setupInfo.ActionRepository}' needs to build image '{setupInfo.Dockerfile}'");
imagesToBuild[setupInfo.ActionRepository].Add(action.Id);
imagesToBuildInfo[setupInfo.ActionRepository] = setupInfo;
}
}
}
}
if (imagesToPull.Count > 0)
{
foreach (var imageToPull in imagesToPull)
{
Trace.Info($"{imageToPull.Value.Count} steps need to pull image '{imageToPull.Key}'");
containerSetupSteps.Add(new JobExtensionRunner(runAsync: this.PullActionContainerAsync,
condition: $"{PipelineTemplateConstants.Success}()",
displayName: $"Pull {imageToPull.Key}",
data: new ContainerSetupInfo(imageToPull.Value, imageToPull.Key)));
}
}
if (imagesToBuild.Count > 0)
{
foreach (var imageToBuild in imagesToBuild)
{
var setupInfo = imagesToBuildInfo[imageToBuild.Key];
Trace.Info($"{imageToBuild.Value.Count} steps need to build image from '{setupInfo.Dockerfile}'");
containerSetupSteps.Add(new JobExtensionRunner(runAsync: this.BuildActionContainerAsync,
condition: $"{PipelineTemplateConstants.Success}()",
displayName: $"Build {setupInfo.ActionRepository}",
data: new ContainerSetupInfo(imageToBuild.Value, setupInfo.Dockerfile, setupInfo.WorkingDirectory)));
}
}
#if !OS_LINUX
if (containerSetupSteps.Count > 0)
{
executionContext.Output("Container action is only supported on Linux, skip pull and build docker images.");
containerSetupSteps.Clear();
}
#endif
return(containerSetupSteps);
}
private async Task StartContainerAsync(IExecutionContext executionContext)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNullOrEmpty(executionContext.Container.ContainerImage, nameof(executionContext.Container.ContainerImage));
// Check docker client/server version
DockerVersion dockerVersion = await _dockerManger.DockerVersion(executionContext);
ArgUtil.NotNull(dockerVersion.ServerVersion, nameof(dockerVersion.ServerVersion));
ArgUtil.NotNull(dockerVersion.ClientVersion, nameof(dockerVersion.ClientVersion));
Version requiredDockerVersion = new Version(17, 3);
if (dockerVersion.ServerVersion < requiredDockerVersion)
{
throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerServerVersion", requiredDockerVersion, _dockerManger.DockerPath, dockerVersion.ServerVersion));
}
if (dockerVersion.ClientVersion < requiredDockerVersion)
{
throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerClientVersion", requiredDockerVersion, _dockerManger.DockerPath, dockerVersion.ClientVersion));
}
// Pull down docker image
int pullExitCode = await _dockerManger.DockerPull(executionContext, executionContext.Container.ContainerImage);
if (pullExitCode != 0)
{
throw new InvalidOperationException($"Docker pull fail with exit code {pullExitCode}");
}
// Mount folder into container
executionContext.Container.MountVolumes.Add(new MountVolume(executionContext.Variables.System_DefaultWorkingDirectory));
executionContext.Container.MountVolumes.Add(new MountVolume(executionContext.Variables.Agent_TempDirectory));
executionContext.Container.MountVolumes.Add(new MountVolume(executionContext.Variables.Agent_ToolsDirectory));
executionContext.Container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), true));
executionContext.Container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tasks), true));
// Ensure .taskkey file exist so we can mount it.
string taskKeyFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), ".taskkey");
if (!File.Exists(taskKeyFile))
{
File.WriteAllText(taskKeyFile, string.Empty);
}
executionContext.Container.MountVolumes.Add(new MountVolume(taskKeyFile));
executionContext.Container.ContainerId = await _dockerManger.DockerCreate(executionContext, executionContext.Container.ContainerImage, executionContext.Container.MountVolumes);
ArgUtil.NotNullOrEmpty(executionContext.Container.ContainerId, nameof(executionContext.Container.ContainerId));
// Get current username
executionContext.Container.CurrentUserName = (await ExecuteCommandAsync(executionContext, "whoami", string.Empty)).FirstOrDefault();
ArgUtil.NotNullOrEmpty(executionContext.Container.CurrentUserName, nameof(executionContext.Container.CurrentUserName));
// Get current userId
executionContext.Container.CurrentUserId = (await ExecuteCommandAsync(executionContext, "id", $"-u {executionContext.Container.CurrentUserName}")).FirstOrDefault();
ArgUtil.NotNullOrEmpty(executionContext.Container.CurrentUserId, nameof(executionContext.Container.CurrentUserId));
int startExitCode = await _dockerManger.DockerStart(executionContext, executionContext.Container.ContainerId);
if (startExitCode != 0)
{
throw new InvalidOperationException($"Docker start fail with exit code {startExitCode}");
}
// Ensure bash exist in the image
int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, executionContext.Container.ContainerId, string.Empty, $"which bash");
if (execWhichBashExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execWhichBashExitCode}");
}
// Create an user with same uid as the agent run as user inside the container.
// All command execute in docker will run as Root by default,
// this will cause the agent on the host machine doesn't have permission to any new file/folder created inside the container.
// So, we create a user account with same UID inside the container and let all docker exec command run as that user.
int execUseraddExitCode = await _dockerManger.DockerExec(executionContext, executionContext.Container.ContainerId, string.Empty, $"useradd -m -u {executionContext.Container.CurrentUserId} {executionContext.Container.CurrentUserName}_VSTSContainer");
if (execUseraddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execUseraddExitCode}");
}
}
private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, Pipelines.ActionStep repositoryAction)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
var repositoryReference = repositoryAction.Reference as Pipelines.RepositoryPathReference;
ArgUtil.NotNull(repositoryReference, nameof(repositoryReference));
if (string.Equals(repositoryReference.RepositoryType, Pipelines.PipelineConstants.SelfAlias, StringComparison.OrdinalIgnoreCase))
{
Trace.Info($"Repository action is in 'self' repository.");
return;
}
if (!string.Equals(repositoryReference.RepositoryType, Pipelines.RepositoryTypes.GitHub, StringComparison.OrdinalIgnoreCase))
{
throw new NotSupportedException(repositoryReference.RepositoryType);
}
ArgUtil.NotNullOrEmpty(repositoryReference.Name, nameof(repositoryReference.Name));
ArgUtil.NotNullOrEmpty(repositoryReference.Ref, nameof(repositoryReference.Ref));
string destDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), repositoryReference.Name.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), repositoryReference.Ref);
string watermarkFile = destDirectory + ".completed";
if (File.Exists(watermarkFile))
{
executionContext.Debug($"Action '{repositoryReference.Name}@{repositoryReference.Ref}' already downloaded at '{destDirectory}'.");
return;
}
else
{
// make sure we get a clean folder ready to use.
IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken);
Directory.CreateDirectory(destDirectory);
executionContext.Output($"Download action repository '{repositoryReference.Name}@{repositoryReference.Ref}'");
}
#if OS_WINDOWS
string archiveLink = $"https://api.github.com/repos/{repositoryReference.Name}/zipball/{repositoryReference.Ref}";
#else
string archiveLink = $"https://api.github.com/repos/{repositoryReference.Name}/tarball/{repositoryReference.Ref}";
#endif
Trace.Info($"Download archive '{archiveLink}' to '{destDirectory}'.");
//download and extract action in a temp folder and rename it on success
string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), "_temp_" + Guid.NewGuid());
Directory.CreateDirectory(tempDirectory);
#if OS_WINDOWS
string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.zip");
#else
string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.tar.gz");
#endif
Trace.Info($"Save archive '{archiveLink}' into {archiveFile}.");
try
{
int retryCount = 0;
// Allow up to 20 * 60s for any action to be downloaded from github graph.
int timeoutSeconds = 20 * 60;
while (retryCount < 3)
{
using (var actionDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
using (var actionDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(actionDownloadTimeout.Token, executionContext.CancellationToken))
{
try
{
//open zip stream in async mode
using (FileStream fs = new FileStream(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true))
using (var httpClientHandler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(httpClientHandler))
{
var configurationStore = HostContext.GetService <IConfigurationStore>();
var isHostedServer = configurationStore.GetSettings().IsHostedServer;
if (isHostedServer)
{
var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN");
if (string.IsNullOrEmpty(authToken))
{
// TODO: Deprecate the PREVIEW_ACTION_TOKEN
authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN");
}
if (!string.IsNullOrEmpty(authToken))
{
HostContext.SecretMasker.AddValue(authToken);
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"PAT:{authToken}"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
}
else
{
var accessToken = executionContext.GetGitHubContext("token");
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
}
}
else
{
// Intentionally empty. Temporary for GHES alpha release, download from dotcom unauthenticated.
}
httpClient.DefaultRequestHeaders.UserAgent.Add(HostContext.UserAgent);
using (var result = await httpClient.GetStreamAsync(archiveLink))
{
await result.CopyToAsync(fs, _defaultCopyBufferSize, actionDownloadCancellation.Token);
await fs.FlushAsync(actionDownloadCancellation.Token);
// download succeed, break out the retry loop.
break;
}
}
}
catch (OperationCanceledException) when(executionContext.CancellationToken.IsCancellationRequested)
{
Trace.Info($"Action download has been cancelled.");
throw;
}
catch (Exception ex) when(retryCount < 2)
{
retryCount++;
Trace.Error($"Fail to download archive '{archiveLink}' -- Attempt: {retryCount}");
Trace.Error(ex);
if (actionDownloadTimeout.Token.IsCancellationRequested)
{
// action download didn't finish within timeout
executionContext.Warning($"Action '{archiveLink}' didn't finish download within {timeoutSeconds} seconds.");
}
else
{
executionContext.Warning($"Failed to download action '{archiveLink}'. Error {ex.Message}");
}
}
}
if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("_GITHUB_ACTION_DOWNLOAD_NO_BACKOFF")))
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30));
executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
ArgUtil.NotNullOrEmpty(archiveFile, nameof(archiveFile));
executionContext.Debug($"Download '{archiveLink}' to '{archiveFile}'");
var stagingDirectory = Path.Combine(tempDirectory, "_staging");
Directory.CreateDirectory(stagingDirectory);
#if OS_WINDOWS
ZipFile.ExtractToDirectory(archiveFile, stagingDirectory);
#else
string tar = WhichUtil.Which("tar", require: true, trace: Trace);
// tar -xzf
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Info(args.Data);
}
});
processInvoker.ErrorDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Error(args.Data);
}
});
int exitCode = await processInvoker.ExecuteAsync(stagingDirectory, tar, $"-xzf \"{archiveFile}\"", null, executionContext.CancellationToken);
if (exitCode != 0)
{
throw new NotSupportedException($"Can't use 'tar -xzf' extract archive file: {archiveFile}. return code: {exitCode}.");
}
}
#endif
// repository archive from github always contains a nested folder
var subDirectories = new DirectoryInfo(stagingDirectory).GetDirectories();
if (subDirectories.Length != 1)
{
throw new InvalidOperationException($"'{archiveFile}' contains '{subDirectories.Length}' directories");
}
else
{
executionContext.Debug($"Unwrap '{subDirectories[0].Name}' to '{destDirectory}'");
IOUtil.CopyDirectory(subDirectories[0].FullName, destDirectory, executionContext.CancellationToken);
}
Trace.Verbose("Create watermark file indicate action download succeed.");
File.WriteAllText(watermarkFile, DateTime.UtcNow.ToString());
executionContext.Debug($"Archive '{archiveFile}' has been unzipped into '{destDirectory}'.");
Trace.Info("Finished getting action repository.");
}
finally
{
try
{
//if the temp folder wasn't moved -> wipe it
if (Directory.Exists(tempDirectory))
{
Trace.Verbose("Deleting action temp folder: {0}", tempDirectory);
IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast.
}
}
catch (Exception ex)
{
//it is not critical if we fail to delete the temp folder
Trace.Warning("Failed to delete temp folder '{0}'. Exception: {1}", tempDirectory, ex);
}
}
}
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously (method has async logic on only certain platforms)
public async Task RunAsync(ActionRunStage stage)
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(Data, nameof(Data));
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
#if OS_WINDOWS || OS_OSX
throw new NotSupportedException($"Container action is only supported on Linux");
#else
// Update the env dictionary.
AddInputsToEnvironment();
var dockerManger = HostContext.GetService <IDockerCommandManager>();
// container image haven't built/pull
if (Data.Image.StartsWith("docker://", StringComparison.OrdinalIgnoreCase))
{
Data.Image = Data.Image.Substring("docker://".Length);
}
else if (Data.Image.EndsWith("Dockerfile") || Data.Image.EndsWith("dockerfile"))
{
// ensure docker file exist
var dockerFile = Path.Combine(ActionDirectory, Data.Image);
ArgUtil.File(dockerFile, nameof(Data.Image));
ExecutionContext.Output($"Dockerfile for action: '{dockerFile}'.");
var imageName = $"{dockerManger.DockerInstanceLabel}:{ExecutionContext.Id.ToString("N")}";
var buildExitCode = await dockerManger.DockerBuild(ExecutionContext, ExecutionContext.GetGitHubContext("workspace"), Directory.GetParent(dockerFile).FullName, imageName);
if (buildExitCode != 0)
{
throw new InvalidOperationException($"Docker build failed with exit code {buildExitCode}");
}
Data.Image = imageName;
}
// run container
var container = new ContainerInfo()
{
ContainerImage = Data.Image,
ContainerName = ExecutionContext.Id.ToString("N"),
ContainerDisplayName = $"{Pipelines.Validation.NameValidation.Sanitize(Data.Image)}_{Guid.NewGuid().ToString("N").Substring(0, 6)}",
};
if (stage == ActionRunStage.Main)
{
if (!string.IsNullOrEmpty(Data.EntryPoint))
{
// use entrypoint from action.yml
container.ContainerEntryPoint = Data.EntryPoint;
}
else
{
// use entrypoint input, this is for action v1 which doesn't have action.yml
container.ContainerEntryPoint = Inputs.GetValueOrDefault("entryPoint");
}
}
else if (stage == ActionRunStage.Post)
{
container.ContainerEntryPoint = Data.Cleanup;
}
// create inputs context for template evaluation
var inputsContext = new DictionaryContextData();
if (this.Inputs != null)
{
foreach (var input in Inputs)
{
inputsContext.Add(input.Key, new StringContextData(input.Value));
}
}
var evaluateContext = new Dictionary <string, PipelineContextData>(StringComparer.OrdinalIgnoreCase);
evaluateContext["inputs"] = inputsContext;
var manifestManager = HostContext.GetService <IActionManifestManager>();
if (Data.Arguments != null)
{
container.ContainerEntryPointArgs = "";
var evaluatedArgs = manifestManager.EvaluateContainerArguments(ExecutionContext, Data.Arguments, evaluateContext);
foreach (var arg in evaluatedArgs)
{
if (!string.IsNullOrEmpty(arg))
{
container.ContainerEntryPointArgs = container.ContainerEntryPointArgs + $" \"{arg.Replace("\"", "\\\"")}\"";
}
else
{
container.ContainerEntryPointArgs = container.ContainerEntryPointArgs + " \"\"";
}
}
}
else
{
container.ContainerEntryPointArgs = Inputs.GetValueOrDefault("args");
}
if (Data.Environment != null)
{
var evaluatedEnv = manifestManager.EvaluateContainerEnvironment(ExecutionContext, Data.Environment, evaluateContext);
foreach (var env in evaluatedEnv)
{
if (!this.Environment.ContainsKey(env.Key))
{
this.Environment[env.Key] = env.Value;
}
}
}
if (ExecutionContext.JobContext.Container.TryGetValue("network", out var networkContextData) && networkContextData is StringContextData networkStringData)
{
container.ContainerNetwork = networkStringData.ToString();
}
var defaultWorkingDirectory = ExecutionContext.GetGitHubContext("workspace");
var tempDirectory = HostContext.GetDirectory(WellKnownDirectory.Temp);
ArgUtil.NotNullOrEmpty(defaultWorkingDirectory, nameof(defaultWorkingDirectory));
ArgUtil.NotNullOrEmpty(tempDirectory, nameof(tempDirectory));
var tempHomeDirectory = Path.Combine(tempDirectory, "_github_home");
Directory.CreateDirectory(tempHomeDirectory);
this.Environment["HOME"] = tempHomeDirectory;
var tempWorkflowDirectory = Path.Combine(tempDirectory, "_github_workflow");
ArgUtil.Directory(tempWorkflowDirectory, nameof(tempWorkflowDirectory));
container.MountVolumes.Add(new MountVolume("/var/run/docker.sock", "/var/run/docker.sock"));
container.MountVolumes.Add(new MountVolume(tempHomeDirectory, "/github/home"));
container.MountVolumes.Add(new MountVolume(tempWorkflowDirectory, "/github/workflow"));
container.MountVolumes.Add(new MountVolume(defaultWorkingDirectory, "/github/workspace"));
container.AddPathTranslateMapping(tempHomeDirectory, "/github/home");
container.AddPathTranslateMapping(tempWorkflowDirectory, "/github/workflow");
container.AddPathTranslateMapping(defaultWorkingDirectory, "/github/workspace");
container.ContainerWorkDirectory = "/github/workspace";
// expose context to environment
foreach (var context in ExecutionContext.ExpressionValues)
{
if (context.Value is IEnvironmentContextData runtimeContext && runtimeContext != null)
{
foreach (var env in runtimeContext.GetRuntimeEnvironmentVariables())
{
Environment[env.Key] = env.Value;
}
}
}
// Add Actions Runtime server info
var systemConnection = ExecutionContext.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
Environment["ACTIONS_RUNTIME_URL"] = systemConnection.Url.AbsoluteUri;
Environment["ACTIONS_RUNTIME_TOKEN"] = systemConnection.Authorization.Parameters[EndpointAuthorizationParameters.AccessToken];
if (systemConnection.Data.TryGetValue("CacheServerUrl", out var cacheUrl) && !string.IsNullOrEmpty(cacheUrl))
{
Environment["ACTIONS_CACHE_URL"] = cacheUrl;
}
foreach (var variable in this.Environment)
{
container.ContainerEnvironmentVariables[variable.Key] = container.TranslateToContainerPath(variable.Value);
}
using (var stdoutManager = new OutputManager(ExecutionContext, ActionCommandManager))
using (var stderrManager = new OutputManager(ExecutionContext, ActionCommandManager))
{
var runExitCode = await dockerManger.DockerRun(ExecutionContext, container, stdoutManager.OnDataReceived, stderrManager.OnDataReceived);
if (runExitCode != 0)
{
ExecutionContext.Error($"Docker run failed with exit code {runExitCode}");
ExecutionContext.Result = TaskResult.Failed;
}
}
#endif
}
private ActionContainer PrepareRepositoryActionAsync(IExecutionContext executionContext, Pipelines.ActionStep repositoryAction)
{
var repositoryReference = repositoryAction.Reference as Pipelines.RepositoryPathReference;
if (string.Equals(repositoryReference.RepositoryType, Pipelines.PipelineConstants.SelfAlias, StringComparison.OrdinalIgnoreCase))
{
Trace.Info($"Repository action is in 'self' repository.");
return(null);
}
var setupInfo = new ActionContainer();
string destDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), repositoryReference.Name.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), repositoryReference.Ref);
string actionEntryDirectory = destDirectory;
string dockerFileRelativePath = repositoryReference.Name;
ArgUtil.NotNull(repositoryReference, nameof(repositoryReference));
if (!string.IsNullOrEmpty(repositoryReference.Path))
{
actionEntryDirectory = Path.Combine(destDirectory, repositoryReference.Path);
dockerFileRelativePath = $"{dockerFileRelativePath}/{repositoryReference.Path}";
setupInfo.ActionRepository = $"{repositoryReference.Name}/{repositoryReference.Path}@{repositoryReference.Ref}";
}
else
{
setupInfo.ActionRepository = $"{repositoryReference.Name}@{repositoryReference.Ref}";
}
// find the docker file or action.yml file
var dockerFile = Path.Combine(actionEntryDirectory, "Dockerfile");
var dockerFileLowerCase = Path.Combine(actionEntryDirectory, "dockerfile");
var actionManifest = Path.Combine(actionEntryDirectory, Constants.Path.ActionManifestYmlFile);
var actionManifestYaml = Path.Combine(actionEntryDirectory, Constants.Path.ActionManifestYamlFile);
if (File.Exists(actionManifest) || File.Exists(actionManifestYaml))
{
executionContext.Debug($"action.yml for action: '{actionManifest}'.");
var manifestManager = HostContext.GetService <IActionManifestManager>();
ActionDefinitionData actionDefinitionData = null;
if (File.Exists(actionManifest))
{
actionDefinitionData = manifestManager.Load(executionContext, actionManifest);
}
else
{
actionDefinitionData = manifestManager.Load(executionContext, actionManifestYaml);
}
if (actionDefinitionData.Execution.ExecutionType == ActionExecutionType.Container)
{
var containerAction = actionDefinitionData.Execution as ContainerActionExecutionData;
if (containerAction.Image.EndsWith("Dockerfile") || containerAction.Image.EndsWith("dockerfile"))
{
var dockerFileFullPath = Path.Combine(actionEntryDirectory, containerAction.Image);
executionContext.Debug($"Dockerfile for action: '{dockerFileFullPath}'.");
setupInfo.Dockerfile = dockerFileFullPath;
setupInfo.WorkingDirectory = destDirectory;
return(setupInfo);
}
else if (containerAction.Image.StartsWith("docker://", StringComparison.OrdinalIgnoreCase))
{
var actionImage = containerAction.Image.Substring("docker://".Length);
executionContext.Debug($"Container image for action: '{actionImage}'.");
setupInfo.Image = actionImage;
return(setupInfo);
}
else
{
throw new NotSupportedException($"'{containerAction.Image}' should be either '[path]/Dockerfile' or 'docker://image[:tag]'.");
}
}
else if (actionDefinitionData.Execution.ExecutionType == ActionExecutionType.NodeJS)
{
Trace.Info($"Action node.js file: {(actionDefinitionData.Execution as NodeJSActionExecutionData).Script}, no more preparation.");
return(null);
}
else if (actionDefinitionData.Execution.ExecutionType == ActionExecutionType.Plugin)
{
Trace.Info($"Action plugin: {(actionDefinitionData.Execution as PluginActionExecutionData).Plugin}, no more preparation.");
return(null);
}
else
{
throw new NotSupportedException(actionDefinitionData.Execution.ExecutionType.ToString());
}
}
else if (File.Exists(dockerFile))
{
executionContext.Debug($"Dockerfile for action: '{dockerFile}'.");
setupInfo.Dockerfile = dockerFile;
setupInfo.WorkingDirectory = destDirectory;
return(setupInfo);
}
else if (File.Exists(dockerFileLowerCase))
{
executionContext.Debug($"Dockerfile for action: '{dockerFileLowerCase}'.");
setupInfo.Dockerfile = dockerFileLowerCase;
setupInfo.WorkingDirectory = destDirectory;
return(setupInfo);
}
else
{
var fullPath = IOUtil.ResolvePath(actionEntryDirectory, "."); // resolve full path without access filesystem.
throw new InvalidOperationException($"Can't find 'action.yml', 'action.yaml' or 'Dockerfile' under '{fullPath}'. Did you forget to run actions/checkout before running your local action?");
}
}
private async Task DownloadArtifacts(IExecutionContext executionContext, IList <AgentArtifactDefinition> agentArtifactDefinitions, string artifactsWorkingFolder)
{
Trace.Entering();
foreach (AgentArtifactDefinition agentArtifactDefinition in agentArtifactDefinitions)
{
// We don't need to check if its old style artifact anymore. All the build data has been fixed and all the build artifact has Alias now.
ArgUtil.NotNullOrEmpty(agentArtifactDefinition.Alias, nameof(agentArtifactDefinition.Alias));
var extensionManager = HostContext.GetService <IExtensionManager>();
IArtifactExtension extension = (extensionManager.GetExtensions <IArtifactExtension>()).FirstOrDefault(x => agentArtifactDefinition.ArtifactType == x.ArtifactType);
if (extension == null)
{
throw new InvalidOperationException(StringUtil.Loc("RMArtifactTypeNotSupported", agentArtifactDefinition.ArtifactType));
}
Trace.Info($"Found artifact extension of type {extension.ArtifactType}");
executionContext.Output(StringUtil.Loc("RMStartArtifactsDownload"));
ArtifactDefinition artifactDefinition = ConvertToArtifactDefinition(agentArtifactDefinition, executionContext, extension);
executionContext.Output(StringUtil.Loc("RMArtifactDownloadBegin", agentArtifactDefinition.Alias, agentArtifactDefinition.ArtifactType));
// Get the local path where this artifact should be downloaded.
string downloadFolderPath = Path.GetFullPath(Path.Combine(artifactsWorkingFolder, agentArtifactDefinition.Alias ?? string.Empty));
// download the artifact to this path.
RetryExecutor retryExecutor = new RetryExecutor();
retryExecutor.ShouldRetryAction = (ex) =>
{
executionContext.Output(StringUtil.Loc("RMErrorDuringArtifactDownload", ex));
bool retry = true;
if (ex is ArtifactDownloadException)
{
retry = false;
}
else
{
executionContext.Output(StringUtil.Loc("RMRetryingArtifactDownload"));
Trace.Warning(ex.ToString());
}
return(retry);
};
await retryExecutor.ExecuteAsync(
async() =>
{
var releaseFileSystemManager = HostContext.GetService <IReleaseFileSystemManager>();
executionContext.Output(StringUtil.Loc("RMEnsureArtifactFolderExistsAndIsClean", downloadFolderPath));
try
{
releaseFileSystemManager.CleanupDirectory(downloadFolderPath, executionContext.CancellationToken);
}
catch (Exception ex) when(ex is DirectoryNotFoundException || ex is UnauthorizedAccessException)
{
throw new ArtifactCleanupFailedException(StringUtil.Loc("FailedCleaningupRMArtifactDirectory", downloadFolderPath), ex);
}
await extension.DownloadAsync(executionContext, artifactDefinition, downloadFolderPath);
});
executionContext.Output(StringUtil.Loc("RMArtifactDownloadFinished", agentArtifactDefinition.Alias));
}
executionContext.Output(StringUtil.Loc("RMArtifactsDownloadFinished"));
}
public async Task <int> RunAsync(string pipeIn, string pipeOut)
{
// Validate args.
ArgUtil.NotNullOrEmpty(pipeIn, nameof(pipeIn));
ArgUtil.NotNullOrEmpty(pipeOut, nameof(pipeOut));
var agentWebProxy = HostContext.GetService <IVstsAgentWebProxy>();
VssHttpMessageHandler.DefaultWebProxy = agentWebProxy;
var jobRunner = HostContext.CreateService <IJobRunner>();
using (var channel = HostContext.CreateService <IProcessChannel>())
using (var jobRequestCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(HostContext.AgentShutdownToken))
using (var channelTokenSource = new CancellationTokenSource())
{
// Start the channel.
channel.StartClient(pipeIn, pipeOut);
// Wait for up to 30 seconds for a message from the channel.
Trace.Info("Waiting to receive the job message from the channel.");
WorkerMessage channelMessage;
using (var csChannelMessage = new CancellationTokenSource(_workerStartTimeout))
{
channelMessage = await channel.ReceiveAsync(csChannelMessage.Token);
}
// Deserialize the job message.
Trace.Info("Message received.");
ArgUtil.Equal(MessageType.NewJobRequest, channelMessage.MessageType, nameof(channelMessage.MessageType));
ArgUtil.NotNullOrEmpty(channelMessage.Body, nameof(channelMessage.Body));
var jobMessage = JsonUtility.FromString <AgentJobRequestMessage>(channelMessage.Body);
ArgUtil.NotNull(jobMessage, nameof(jobMessage));
// Initialize the secret masker and set the thread culture.
InitializeSecretMasker(jobMessage);
SetCulture(jobMessage);
// Start the job.
Trace.Info($"Job message:{Environment.NewLine} {StringUtil.ConvertToJson(jobMessage)}");
Task <TaskResult> jobRunnerTask = jobRunner.RunAsync(jobMessage, jobRequestCancellationToken.Token);
// Start listening for a cancel message from the channel.
Trace.Info("Listening for cancel message from the channel.");
Task <WorkerMessage> channelTask = channel.ReceiveAsync(channelTokenSource.Token);
// Wait for one of the tasks to complete.
Trace.Info("Waiting for the job to complete or for a cancel message from the channel.");
Task.WaitAny(jobRunnerTask, channelTask);
// Handle if the job completed.
if (jobRunnerTask.IsCompleted)
{
Trace.Info("Job completed.");
channelTokenSource.Cancel(); // Cancel waiting for a message from the channel.
return(TaskResultUtil.TranslateToReturnCode(await jobRunnerTask));
}
// Otherwise a cancel message was received from the channel.
Trace.Info("Cancellation/Shutdown message received.");
channelMessage = await channelTask;
switch (channelMessage.MessageType)
{
case MessageType.CancelRequest:
jobRequestCancellationToken.Cancel(); // Expire the host cancellation token.
break;
case MessageType.AgentShutdown:
HostContext.ShutdownAgent(ShutdownReason.UserCancelled);
break;
case MessageType.OperatingSystemShutdown:
HostContext.ShutdownAgent(ShutdownReason.OperatingSystemShutdown);
break;
default:
throw new ArgumentOutOfRangeException(nameof(channelMessage.MessageType), channelMessage.MessageType, nameof(channelMessage.MessageType));
}
// Await the job.
return(TaskResultUtil.TranslateToReturnCode(await jobRunnerTask));
}
}
private async Task PullContainerAsync(IExecutionContext executionContext, ContainerInfo container)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(container, nameof(container));
ArgUtil.NotNullOrEmpty(container.ContainerImage, nameof(container.ContainerImage));
Trace.Info($"Container name: {container.ContainerName}");
Trace.Info($"Container image: {container.ContainerImage}");
Trace.Info($"Container registry: {container.ContainerRegistryEndpoint.ToString()}");
Trace.Info($"Container options: {container.ContainerCreateOptions}");
Trace.Info($"Skip container image pull: {container.SkipContainerImagePull}");
// Login to private docker registry
string registryServer = string.Empty;
if (container.ContainerRegistryEndpoint != Guid.Empty)
{
var registryEndpoint = executionContext.Endpoints.FirstOrDefault(x => x.Type == "dockerregistry" && x.Id == container.ContainerRegistryEndpoint);
ArgUtil.NotNull(registryEndpoint, nameof(registryEndpoint));
string username = string.Empty;
string password = string.Empty;
string registryType = string.Empty;
registryEndpoint.Data?.TryGetValue("registrytype", out registryType);
if (string.Equals(registryType, "ACR", StringComparison.OrdinalIgnoreCase))
{
string loginServer = string.Empty;
registryEndpoint.Authorization?.Parameters?.TryGetValue("loginServer", out loginServer);
if (loginServer != null)
{
loginServer = loginServer.ToLower();
}
registryEndpoint.Authorization?.Parameters?.TryGetValue("serviceprincipalid", out username);
registryEndpoint.Authorization?.Parameters?.TryGetValue("serviceprincipalkey", out password);
registryServer = $"https://{loginServer}";
}
else
{
registryEndpoint.Authorization?.Parameters?.TryGetValue("registry", out registryServer);
registryEndpoint.Authorization?.Parameters?.TryGetValue("username", out username);
registryEndpoint.Authorization?.Parameters?.TryGetValue("password", out password);
}
ArgUtil.NotNullOrEmpty(registryServer, nameof(registryServer));
ArgUtil.NotNullOrEmpty(username, nameof(username));
ArgUtil.NotNullOrEmpty(password, nameof(password));
int loginExitCode = await _dockerManger.DockerLogin(executionContext, registryServer, username, password);
if (loginExitCode != 0)
{
throw new InvalidOperationException($"Docker login fail with exit code {loginExitCode}");
}
}
try
{
if (!container.SkipContainerImagePull)
{
if (!string.IsNullOrEmpty(registryServer) &&
registryServer.IndexOf("index.docker.io", StringComparison.OrdinalIgnoreCase) < 0)
{
var registryServerUri = new Uri(registryServer);
if (!container.ContainerImage.StartsWith(registryServerUri.Authority, StringComparison.OrdinalIgnoreCase))
{
container.ContainerImage = $"{registryServerUri.Authority}/{container.ContainerImage}";
}
}
// Pull down docker image with retry up to 3 times
int retryCount = 0;
int pullExitCode = 0;
while (retryCount < 3)
{
pullExitCode = await _dockerManger.DockerPull(executionContext, container.ContainerImage);
if (pullExitCode == 0)
{
break;
}
else
{
retryCount++;
if (retryCount < 3)
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
executionContext.Warning($"Docker pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
}
if (retryCount == 3 && pullExitCode != 0)
{
throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}");
}
}
if (PlatformUtil.RunningOnMacOS)
{
container.ImageOS = PlatformUtil.OS.Linux;
}
// if running on Windows, and attempting to run linux container, require container to have node
else if (PlatformUtil.RunningOnWindows)
{
string containerOS = await _dockerManger.DockerInspect(context : executionContext,
dockerObject : container.ContainerImage,
options : $"--format=\"{{{{.Os}}}}\"");
if (string.Equals("linux", containerOS, StringComparison.OrdinalIgnoreCase))
{
container.ImageOS = PlatformUtil.OS.Linux;
}
}
}
finally
{
// Logout for private registry
if (!string.IsNullOrEmpty(registryServer))
{
int logoutExitCode = await _dockerManger.DockerLogout(executionContext, registryServer);
if (logoutExitCode != 0)
{
executionContext.Error($"Docker logout fail with exit code {logoutExitCode}");
}
}
}
}
public virtual async Task GetSourceAsync(IExecutionContext executionContext, ServiceEndpoint endpoint, CancellationToken cancellationToken)
{
Trace.Entering();
ArgUtil.NotNull(endpoint, nameof(endpoint));
executionContext.Output($"Syncing repository: {endpoint.Name} (Git)");
_gitCommandManager = HostContext.GetService <IGitCommandManager>();
await _gitCommandManager.LoadGitExecutionInfo(executionContext);
string targetPath = executionContext.Variables.Get(Constants.Variables.Build.SourcesDirectory);
string sourceBranch = executionContext.Variables.Get(Constants.Variables.Build.SourceBranch);
string sourceVersion = executionContext.Variables.Get(Constants.Variables.Build.SourceVersion);
bool clean = false;
if (endpoint.Data.ContainsKey(WellKnownEndpointData.Clean))
{
clean = StringUtil.ConvertToBoolean(endpoint.Data[WellKnownEndpointData.Clean]);
}
bool checkoutSubmodules = false;
if (endpoint.Data.ContainsKey(WellKnownEndpointData.CheckoutSubmodules))
{
checkoutSubmodules = StringUtil.ConvertToBoolean(endpoint.Data[WellKnownEndpointData.CheckoutSubmodules]);
}
bool exposeCred = executionContext.Variables.GetBoolean(Constants.Variables.System.EnableAccessToken) ?? false;
Trace.Info($"Repository url={endpoint.Url}");
Trace.Info($"targetPath={targetPath}");
Trace.Info($"sourceBranch={sourceBranch}");
Trace.Info($"sourceVersion={sourceVersion}");
Trace.Info($"clean={clean}");
Trace.Info($"checkoutSubmodules={checkoutSubmodules}");
Trace.Info($"exposeCred={exposeCred}");
// retrieve credential from endpoint.
Uri repositoryUrl = endpoint.Url;
if (!repositoryUrl.IsAbsoluteUri)
{
throw new InvalidOperationException("Repository url need to be an absolute uri.");
}
string username = string.Empty;
string password = string.Empty;
if (endpoint.Authorization != null)
{
switch (endpoint.Authorization.Scheme)
{
case EndpointAuthorizationSchemes.OAuth:
username = EndpointAuthorizationSchemes.OAuth;
if (!endpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.AccessToken, out password))
{
password = string.Empty;
}
break;
case EndpointAuthorizationSchemes.UsernamePassword:
if (!endpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.Username, out username))
{
// leave the username as empty, the username might in the url, like: http://[email protected]
username = string.Empty;
}
if (!endpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.Password, out password))
{
// we have username, but no password
password = string.Empty;
}
break;
default:
executionContext.Warning($"Unsupport endpoint authorization schemes: {endpoint.Authorization.Scheme}");
break;
}
}
// Check the current contents of the root folder to see if there is already a repo
// If there is a repo, see if it matches the one we are expecting to be there based on the remote fetch url
// if the repo is not what we expect, remove the folder
if (!await IsRepositoryOriginUrlMatch(executionContext, targetPath, repositoryUrl))
{
// Delete source folder
IOUtil.DeleteDirectory(targetPath, cancellationToken);
}
else
{
// delete the index.lock file left by previous canceled build or any operation casue git.exe crash last time.
string lockFile = Path.Combine(targetPath, ".git\\index.lock");
if (File.Exists(lockFile))
{
try
{
File.Delete(lockFile);
}
catch (Exception ex)
{
executionContext.Debug($"Unable to delete the index.lock file: {lockFile}");
executionContext.Debug(ex.ToString());
}
}
// When repo.clean is selected for a git repo, execute git clean -fdx and git reset --hard HEAD on the current repo.
// This will help us save the time to reclone the entire repo.
// If any git commands exit with non-zero return code or any exception happened during git.exe invoke, fall back to delete the repo folder.
if (clean)
{
Boolean softClean = false;
// git clean -fdx
// git reset --hard HEAD
int exitCode_clean = await _gitCommandManager.GitClean(executionContext, targetPath);
if (exitCode_clean != 0)
{
executionContext.Debug($"'git clean -fdx' failed with exit code {exitCode_clean}, this normally caused by:\n 1) Path too long\n 2) Permission issue\n 3) File in use\nFor futher investigation, manually run 'git clean -fdx' on repo root: {targetPath} after each build.");
}
else
{
int exitCode_reset = await _gitCommandManager.GitReset(executionContext, targetPath);
if (exitCode_reset != 0)
{
executionContext.Debug($"'git reset --hard HEAD' failed with exit code {exitCode_reset}\nFor futher investigation, manually run 'git reset --hard HEAD' on repo root: {targetPath} after each build.");
}
else
{
softClean = true;
}
}
if (!softClean)
{
//fall back
executionContext.Warning("Unable to run \"git clean -fdx\" and \"git reset --hard HEAD\" successfully, delete source folder instead.");
IOUtil.DeleteDirectory(targetPath, cancellationToken);
}
}
}
// if the folder is missing, create it
if (!Directory.Exists(targetPath))
{
Directory.CreateDirectory(targetPath);
}
// if the folder contains a .git folder, it means the folder contains a git repo that matches the remote url and in a clean state.
// we will run git fetch to update the repo.
if (!Directory.Exists(Path.Combine(targetPath, ".git")))
{
// init git repository
int exitCode_init = await _gitCommandManager.GitInit(executionContext, targetPath);
if (exitCode_init != 0)
{
throw new InvalidOperationException($"Unable to use git.exe init repository under {targetPath}, 'git init' failed with exit code: {exitCode_init}");
}
int exitCode_addremote = await _gitCommandManager.GitRemoteAdd(executionContext, targetPath, "origin", repositoryUrl.AbsoluteUri);
if (exitCode_addremote != 0)
{
throw new InvalidOperationException($"Unable to use git.exe add remote 'origin', 'git remote add' failed with exit code: {exitCode_addremote}");
}
}
cancellationToken.ThrowIfCancellationRequested();
executionContext.Progress(0, "Starting fetch...");
// disable git auto gc
int exitCode_disableGC = await _gitCommandManager.GitDisableAutoGC(executionContext, targetPath);
if (exitCode_disableGC != 0)
{
executionContext.Warning("Unable turn off git auto garbage collection, git fetch operation may trigger auto garbage collection which will affect the performence of fetching.");
}
// inject credential into fetch url
executionContext.Debug("Inject credential into git remote url.");
Uri urlWithCred = null;
urlWithCred = GetCredentialEmbeddedRepoUrl(repositoryUrl, username, password);
// inject credential into fetch url
executionContext.Debug("Inject credential into git remote fetch url.");
int exitCode_seturl = await _gitCommandManager.GitRemoteSetUrl(executionContext, targetPath, "origin", urlWithCred.AbsoluteUri);
if (exitCode_seturl != 0)
{
throw new InvalidOperationException($"Unable to use git.exe inject credential to git remote fetch url, 'git remote set-url' failed with exit code: {exitCode_seturl}");
}
// inject credential into push url
executionContext.Debug("Inject credential into git remote push url.");
exitCode_seturl = await _gitCommandManager.GitRemoteSetPushUrl(executionContext, targetPath, "origin", urlWithCred.AbsoluteUri);
if (exitCode_seturl != 0)
{
throw new InvalidOperationException($"Unable to use git.exe inject credential to git remote push url, 'git remote set-url --push' failed with exit code: {exitCode_seturl}");
}
// If this is a build for a pull request, then include
// the pull request reference as an additional ref.
string fetchSpec = IsPullRequest(sourceBranch) ? StringUtil.Format("+{0}:{1}", sourceBranch, GetRemoteRefName(sourceBranch)) : null;
int exitCode_fetch = await _gitCommandManager.GitFetch(executionContext, targetPath, "origin", new List <string>() { fetchSpec }, string.Empty, cancellationToken);
if (exitCode_fetch != 0)
{
throw new InvalidOperationException($"Git fetch failed with exit code: {exitCode_fetch}");
}
if (!exposeCred)
{
// remove cached credential from origin's fetch/push url.
await RemoveCachedCredential(executionContext, targetPath, repositoryUrl, "origin");
}
// Checkout
// sourceToBuild is used for checkout
// if sourceBranch is a PR branch or sourceVersion is null, make sure branch name is a remote branch. we need checkout to detached head.
// (change refs/heads to refs/remotes/origin, refs/pull to refs/remotes/pull, or leava it as it when the branch name doesn't contain refs/...)
// if sourceVersion provide, just use that for checkout, since when you checkout a commit, it will end up in detached head.
cancellationToken.ThrowIfCancellationRequested();
executionContext.Progress(80, "Starting checkout...");
string sourcesToBuild;
if (IsPullRequest(sourceBranch) || string.IsNullOrEmpty(sourceVersion))
{
sourcesToBuild = GetRemoteRefName(sourceBranch);
}
else
{
sourcesToBuild = sourceVersion;
}
// Finally, checkout the sourcesToBuild (if we didn't find a valid git object this will throw)
int exitCode_checkout = await _gitCommandManager.GitCheckout(executionContext, targetPath, sourcesToBuild, cancellationToken);
if (exitCode_checkout != 0)
{
throw new InvalidOperationException($"Git checkout failed with exit code: {exitCode_checkout}");
}
// Submodule update
if (checkoutSubmodules)
{
cancellationToken.ThrowIfCancellationRequested();
executionContext.Progress(90, "Updating submodules...");
int exitCode_submoduleInit = await _gitCommandManager.GitSubmoduleInit(executionContext, targetPath);
if (exitCode_submoduleInit != 0)
{
throw new InvalidOperationException($"Git submodule init failed with exit code: {exitCode_submoduleInit}");
}
int exitCode_submoduleUpdate = await _gitCommandManager.GitSubmoduleUpdate(executionContext, targetPath, string.Empty, cancellationToken);
if (exitCode_submoduleUpdate != 0)
{
throw new InvalidOperationException($"Git submodule update failed with exit code: {exitCode_submoduleUpdate}");
}
}
}
public async Task StartContainersAsync(IExecutionContext executionContext, object data)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
List <ContainerInfo> containers = data as List <ContainerInfo>;
ArgUtil.NotNull(containers, nameof(containers));
containers = containers.FindAll(c => c != null); // attempt to mitigate issue #11902 filed in azure-pipelines-task repo
// Check whether we are inside a container.
// Our container feature requires to map working directory from host to the container.
// If we are already inside a container, we will not able to find out the real working direcotry path on the host.
if (PlatformUtil.RunningOnRHEL6)
{
// Red Hat and CentOS 6 do not support the container feature
throw new NotSupportedException(StringUtil.Loc("AgentDoesNotSupportContainerFeatureRhel6"));
}
ThrowIfAlreadyInContainer();
ThrowIfWrongWindowsVersion(executionContext);
// Check docker client/server version
DockerVersion dockerVersion = await _dockerManger.DockerVersion(executionContext);
ArgUtil.NotNull(dockerVersion.ServerVersion, nameof(dockerVersion.ServerVersion));
ArgUtil.NotNull(dockerVersion.ClientVersion, nameof(dockerVersion.ClientVersion));
Version requiredDockerEngineAPIVersion = PlatformUtil.RunningOnWindows
? new Version(1, 30) // Docker-EE version 17.6
: new Version(1, 35); // Docker-CE version 17.12
if (dockerVersion.ServerVersion < requiredDockerEngineAPIVersion)
{
throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerServerVersion", requiredDockerEngineAPIVersion, _dockerManger.DockerPath, dockerVersion.ServerVersion));
}
if (dockerVersion.ClientVersion < requiredDockerEngineAPIVersion)
{
throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerClientVersion", requiredDockerEngineAPIVersion, _dockerManger.DockerPath, dockerVersion.ClientVersion));
}
// Clean up containers left by previous runs
executionContext.Debug($"Delete stale containers from previous jobs");
var staleContainers = await _dockerManger.DockerPS(executionContext, $"--all --quiet --no-trunc --filter \"label={_dockerManger.DockerInstanceLabel}\"");
foreach (var staleContainer in staleContainers)
{
int containerRemoveExitCode = await _dockerManger.DockerRemove(executionContext, staleContainer);
if (containerRemoveExitCode != 0)
{
executionContext.Warning($"Delete stale containers failed, docker rm fail with exit code {containerRemoveExitCode} for container {staleContainer}");
}
}
executionContext.Debug($"Delete stale container networks from previous jobs");
int networkPruneExitCode = await _dockerManger.DockerNetworkPrune(executionContext);
if (networkPruneExitCode != 0)
{
executionContext.Warning($"Delete stale container networks failed, docker network prune fail with exit code {networkPruneExitCode}");
}
// We need to pull the containers first before setting up the network
foreach (var container in containers)
{
await PullContainerAsync(executionContext, container);
}
// Create local docker network for this job to avoid port conflict when multiple agents run on same machine.
// All containers within a job join the same network
await CreateContainerNetworkAsync(executionContext, _containerNetwork);
containers.ForEach(container => container.ContainerNetwork = _containerNetwork);
foreach (var container in containers)
{
await StartContainerAsync(executionContext, container);
}
foreach (var container in containers.Where(c => !c.IsJobContainer))
{
await ContainerHealthcheck(executionContext, container);
}
}
public async Task UndoAsync(string localPath)
{
ArgUtil.NotNullOrEmpty(localPath, nameof(localPath));
await RunCommandAsync(FormatFlags.OmitCollectionUrl, "vc", "undo", "/recursive", localPath);
}
