private async Task StartContainerAsync(IExecutionContext executionContext, ContainerInfo container) { Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(container, nameof(container)); ArgUtil.NotNullOrEmpty(container.ContainerImage, nameof(container.ContainerImage)); Trace.Info($"Container name: {container.ContainerName}"); Trace.Info($"Container image: {container.ContainerImage}"); Trace.Info($"Container registry: {container.ContainerRegistryEndpoint.ToString()}"); Trace.Info($"Container options: {container.ContainerCreateOptions}"); Trace.Info($"Skip container image pull: {container.SkipContainerImagePull}"); foreach (var port in container.UserPortMappings) { Trace.Info($"User provided port: {port.Value}"); } foreach (var volume in container.UserMountVolumes) { Trace.Info($"User provided volume: {volume.Value}"); } // Login to private docker registry string registryServer = string.Empty; if (container.ContainerRegistryEndpoint != Guid.Empty) { var registryEndpoint = executionContext.Endpoints.FirstOrDefault(x => x.Type == "dockerregistry" && x.Id == container.ContainerRegistryEndpoint); ArgUtil.NotNull(registryEndpoint, nameof(registryEndpoint)); string username = string.Empty; string password = string.Empty; string registryType = string.Empty; registryEndpoint.Data?.TryGetValue("registrytype", out registryType); if (string.Equals(registryType, "ACR", StringComparison.OrdinalIgnoreCase)) { string loginServer = string.Empty; registryEndpoint.Authorization?.Parameters?.TryGetValue("loginServer", out loginServer); registryEndpoint.Authorization?.Parameters?.TryGetValue("serviceprincipalid", out username); registryEndpoint.Authorization?.Parameters?.TryGetValue("serviceprincipalkey", out password); registryServer = $"https://{loginServer}"; } else { registryEndpoint.Authorization?.Parameters?.TryGetValue("registry", out registryServer); registryEndpoint.Authorization?.Parameters?.TryGetValue("username", out username); registryEndpoint.Authorization?.Parameters?.TryGetValue("password", out password); } ArgUtil.NotNullOrEmpty(registryServer, nameof(registryServer)); ArgUtil.NotNullOrEmpty(username, nameof(username)); ArgUtil.NotNullOrEmpty(password, nameof(password)); int loginExitCode = await _dockerManger.DockerLogin(executionContext, registryServer, username, password); if (loginExitCode != 0) { throw new InvalidOperationException($"Docker login fail with exit code {loginExitCode}"); } } try { if (!container.SkipContainerImagePull) { if (!string.IsNullOrEmpty(registryServer) && registryServer.IndexOf("index.docker.io", StringComparison.OrdinalIgnoreCase) < 0) { var registryServerUri = new Uri(registryServer); if (!container.ContainerImage.StartsWith(registryServerUri.Authority, StringComparison.OrdinalIgnoreCase)) { container.ContainerImage = $"{registryServerUri.Authority}/{container.ContainerImage}"; } } // Pull down docker image with retry up to 3 times int retryCount = 0; int pullExitCode = 0; while (retryCount < 3) { pullExitCode = await _dockerManger.DockerPull(executionContext, container.ContainerImage); if (pullExitCode == 0) { break; } else { retryCount++; if (retryCount < 3) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10)); executionContext.Warning($"Docker pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } } } if (retryCount == 3 && pullExitCode != 0) { throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}"); } } // Mount folder into container #if OS_WINDOWS container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Work), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Work)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools)))); #else string defaultWorkingDirectory = executionContext.Variables.Get(Constants.Variables.System.DefaultWorkingDirectory); if (string.IsNullOrEmpty(defaultWorkingDirectory)) { throw new NotSupportedException(StringUtil.Loc("ContainerJobRequireSystemDefaultWorkDir")); } string workingDirectory = Path.GetDirectoryName(defaultWorkingDirectory.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)); container.MountVolumes.Add(new MountVolume(container.TranslateToHostPath(workingDirectory), workingDirectory)); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Temp), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Temp)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tasks), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tasks)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)), true)); // Ensure .taskkey file exist so we can mount it. string taskKeyFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), ".taskkey"); if (!File.Exists(taskKeyFile)) { File.WriteAllText(taskKeyFile, string.Empty); } container.MountVolumes.Add(new MountVolume(taskKeyFile, container.TranslateToContainerPath(taskKeyFile))); #endif if (container.IsJobContainer) { // See if this container brings its own Node.js container.ContainerBringNodePath = await _dockerManger.DockerInspect(context : executionContext, dockerObject : container.ContainerImage, options : $"--format=\"{{{{index .Config.Labels \\\"{_nodeJsPathLabel}\\\"}}}}\""); string node; if (!string.IsNullOrEmpty(container.ContainerBringNodePath)) { node = container.ContainerBringNodePath; } else { node = container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node", "bin", $"node{IOUtil.ExeExtension}")); } string sleepCommand = $"\"{node}\" -e \"setInterval(function(){{}}, 24 * 60 * 60 * 1000);\""; container.ContainerCommand = sleepCommand; } container.ContainerId = await _dockerManger.DockerCreate(executionContext, container); ArgUtil.NotNullOrEmpty(container.ContainerId, nameof(container.ContainerId)); if (container.IsJobContainer) { executionContext.Variables.Set(Constants.Variables.Agent.ContainerId, container.ContainerId); } // Start container int startExitCode = await _dockerManger.DockerStart(executionContext, container.ContainerId); if (startExitCode != 0) { throw new InvalidOperationException($"Docker start fail with exit code {startExitCode}"); } } finally { // Logout for private registry if (!string.IsNullOrEmpty(registryServer)) { int logoutExitCode = await _dockerManger.DockerLogout(executionContext, registryServer); if (logoutExitCode != 0) { executionContext.Error($"Docker logout fail with exit code {logoutExitCode}"); } } } try { // Make sure container is up and running var psOutputs = await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --filter status=running --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\""); if (psOutputs.FirstOrDefault(x => !string.IsNullOrEmpty(x))?.StartsWith(container.ContainerId) != true) { // container is not up and running, pull docker log for this container. await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\""); int logsExitCode = await _dockerManger.DockerLogs(executionContext, container.ContainerId); if (logsExitCode != 0) { executionContext.Warning($"Docker logs fail with exit code {logsExitCode}"); } executionContext.Warning($"Docker container {container.ContainerId} is not in running state."); } } catch (Exception ex) { // pull container log is best effort. Trace.Error("Catch exception when check container log and container status."); Trace.Error(ex); } // Get port mappings of running container if (executionContext.Container == null && !container.IsJobContainer) { container.AddPortMappings(await _dockerManger.DockerPort(executionContext, container.ContainerId)); foreach (var port in container.PortMappings) { executionContext.Variables.Set( $"{Constants.Variables.Agent.ServicePortPrefix}.{container.ContainerNetworkAlias}.ports.{port.ContainerPort}", $"{port.HostPort}"); } } #if !OS_WINDOWS if (container.IsJobContainer) { // Ensure bash exist in the image int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"sh -c \"command -v bash\""); if (execWhichBashExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execWhichBashExitCode}"); } // Get current username container.CurrentUserName = (await ExecuteCommandAsync(executionContext, "whoami", string.Empty)).FirstOrDefault(); ArgUtil.NotNullOrEmpty(container.CurrentUserName, nameof(container.CurrentUserName)); // Get current userId container.CurrentUserId = (await ExecuteCommandAsync(executionContext, "id", $"-u {container.CurrentUserName}")).FirstOrDefault(); ArgUtil.NotNullOrEmpty(container.CurrentUserId, nameof(container.CurrentUserId)); executionContext.Output(StringUtil.Loc("CreateUserWithSameUIDInsideContainer", container.CurrentUserId)); // Create an user with same uid as the agent run as user inside the container. // All command execute in docker will run as Root by default, // this will cause the agent on the host machine doesn't have permission to any new file/folder created inside the container. // So, we create a user account with same UID inside the container and let all docker exec command run as that user. string containerUserName = string.Empty; // We need to find out whether there is a user with same UID inside the container List <string> userNames = new List <string>(); int execGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"grep {container.CurrentUserId} /etc/passwd | cut -f1 -d:\"", userNames); if (execGrepExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGrepExitCode}"); } if (userNames.Count > 0) { // check all potential username that might match the UID. foreach (string username in userNames) { int execIdExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"id -u {username}"); if (execIdExitCode == 0) { containerUserName = username; break; } } } // Create a new user with same UID if (string.IsNullOrEmpty(containerUserName)) { containerUserName = $"{container.CurrentUserName}_azpcontainer"; int execUseraddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"useradd -m -u {container.CurrentUserId} {containerUserName}"); if (execUseraddExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execUseraddExitCode}"); } } executionContext.Output(StringUtil.Loc("GrantContainerUserSUDOPrivilege", containerUserName)); // Create a new group for giving sudo permission int execGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd azure_pipelines_sudo"); if (execGroupaddExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGroupaddExitCode}"); } // Add the new created user to the new created sudo group. int execUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G azure_pipelines_sudo {containerUserName}"); if (execUsermodExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execUsermodExitCode}"); } // Allow the new sudo group run any sudo command without providing password. int execEchoExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"su -c \"echo '%azure_pipelines_sudo ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers\""); if (execUsermodExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execEchoExitCode}"); } bool setupDockerGroup = executionContext.Variables.GetBoolean("VSTS_SETUP_DOCKERGROUP") ?? StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("VSTS_SETUP_DOCKERGROUP"), true); if (setupDockerGroup) { executionContext.Output(StringUtil.Loc("AllowContainerUserRunDocker", containerUserName)); // Get docker.sock group id on Host string dockerSockGroupId = (await ExecuteCommandAsync(executionContext, "stat", $"-c %g /var/run/docker.sock")).FirstOrDefault(); // We need to find out whether there is a group with same GID inside the container string existingGroupName = null; List <string> groupsOutput = new List <string>(); int execGroupGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"cat /etc/group\"", groupsOutput); if (execGroupGrepExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGroupGrepExitCode}"); } if (groupsOutput.Count > 0) { // check all potential groups that might match the GID. foreach (string groupOutput in groupsOutput) { if (!string.IsNullOrEmpty(groupOutput)) { var groupSegments = groupOutput.Split(':'); if (groupSegments.Length != 4) { Trace.Warning($"Unexpected output from /etc/group: '{groupOutput}'"); } else { // the output of /etc/group should looks like `group:x:gid:` var groupName = groupSegments[0]; var groupId = groupSegments[2]; if (string.Equals(dockerSockGroupId, groupId)) { existingGroupName = groupName; break; } } } } } if (string.IsNullOrEmpty(existingGroupName)) { // create a new group with same gid existingGroupName = "azure_pipelines_docker"; int execDockerGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd -g {dockerSockGroupId} azure_pipelines_docker"); if (execDockerGroupaddExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execDockerGroupaddExitCode}"); } } // Add the new created user to the docker socket group. int execGroupUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G {existingGroupName} {containerUserName}"); if (execGroupUsermodExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGroupUsermodExitCode}"); } } } #endif }
private async Task StartContainerAsync(IExecutionContext executionContext, ContainerInfo container) { Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(container, nameof(container)); ArgUtil.NotNullOrEmpty(container.ContainerImage, nameof(container.ContainerImage)); Trace.Info($"Container name: {container.ContainerName}"); Trace.Info($"Container image: {container.ContainerImage}"); Trace.Info($"Container registry: {container.ContainerRegistryEndpoint.ToString()}"); Trace.Info($"Container options: {container.ContainerCreateOptions}"); Trace.Info($"Skip container image pull: {container.SkipContainerImagePull}"); foreach (var port in container.UserPortMappings) { Trace.Info($"User provided port: {port.Value}"); } foreach (var volume in container.UserMountVolumes) { Trace.Info($"User provided volume: {volume.Value}"); } if (container.ImageOS != PlatformUtil.OS.Windows) { string defaultWorkingDirectory = executionContext.Variables.Get(Constants.Variables.System.DefaultWorkingDirectory); if (string.IsNullOrEmpty(defaultWorkingDirectory)) { throw new NotSupportedException(StringUtil.Loc("ContainerJobRequireSystemDefaultWorkDir")); } string workingDirectory = IOUtil.GetDirectoryName(defaultWorkingDirectory.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), container.ImageOS); string mountWorkingDirectory = container.TranslateToHostPath(workingDirectory); executionContext.Debug($"Default Working Directory {defaultWorkingDirectory}"); executionContext.Debug($"Working Directory {workingDirectory}"); executionContext.Debug($"Mount Working Directory {mountWorkingDirectory}"); container.MountVolumes.Add(new MountVolume(mountWorkingDirectory, workingDirectory)); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Temp), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Temp)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tasks), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tasks)))); } else { container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Work), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Work)))); } container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools)))); bool externalReadOnly = container.ImageOS != PlatformUtil.OS.Windows; // This code was refactored to use PlatformUtils. The previous implementation did not have the externals directory mounted read-only for Windows. // That seems wrong, but to prevent any potential backwards compatibility issues, we are keeping the same logic container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)), externalReadOnly)); if (container.ImageOS != PlatformUtil.OS.Windows) { // Ensure .taskkey file exist so we can mount it. string taskKeyFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), ".taskkey"); if (!File.Exists(taskKeyFile)) { File.WriteAllText(taskKeyFile, string.Empty); } container.MountVolumes.Add(new MountVolume(taskKeyFile, container.TranslateToContainerPath(taskKeyFile))); } if (container.IsJobContainer) { // See if this container brings its own Node.js container.ContainerBringNodePath = await _dockerManger.DockerInspect(context : executionContext, dockerObject : container.ContainerImage, options : $"--format=\"{{{{index .Config.Labels \\\"{_nodeJsPathLabel}\\\"}}}}\""); string node; if (!string.IsNullOrEmpty(container.ContainerBringNodePath)) { node = container.ContainerBringNodePath; } else { node = container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node", "bin", $"node{IOUtil.ExeExtension}")); // if on Mac OS X, require container to have node if (PlatformUtil.RunningOnMacOS) { container.ContainerBringNodePath = "node"; node = container.ContainerBringNodePath; } // if running on Windows, and attempting to run linux container, require container to have node else if (PlatformUtil.RunningOnWindows && container.ImageOS == PlatformUtil.OS.Linux) { container.ContainerBringNodePath = "node"; node = container.ContainerBringNodePath; } } string sleepCommand = $"\"{node}\" -e \"setInterval(function(){{}}, 24 * 60 * 60 * 1000);\""; container.ContainerCommand = sleepCommand; } container.ContainerId = await _dockerManger.DockerCreate(executionContext, container); ArgUtil.NotNullOrEmpty(container.ContainerId, nameof(container.ContainerId)); if (container.IsJobContainer) { executionContext.Variables.Set(Constants.Variables.Agent.ContainerId, container.ContainerId); } // Start container int startExitCode = await _dockerManger.DockerStart(executionContext, container.ContainerId); if (startExitCode != 0) { throw new InvalidOperationException($"Docker start fail with exit code {startExitCode}"); } try { // Make sure container is up and running var psOutputs = await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --filter status=running --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\""); if (psOutputs.FirstOrDefault(x => !string.IsNullOrEmpty(x))?.StartsWith(container.ContainerId) != true) { // container is not up and running, pull docker log for this container. await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\""); int logsExitCode = await _dockerManger.DockerLogs(executionContext, container.ContainerId); if (logsExitCode != 0) { executionContext.Warning($"Docker logs fail with exit code {logsExitCode}"); } executionContext.Warning($"Docker container {container.ContainerId} is not in running state."); } } catch (Exception ex) { // pull container log is best effort. Trace.Error("Catch exception when check container log and container status."); Trace.Error(ex); } // Get port mappings of running container if (executionContext.Container == null && !container.IsJobContainer) { container.AddPortMappings(await _dockerManger.DockerPort(executionContext, container.ContainerId)); foreach (var port in container.PortMappings) { executionContext.Variables.Set( $"{Constants.Variables.Agent.ServicePortPrefix}.{container.ContainerNetworkAlias}.ports.{port.ContainerPort}", $"{port.HostPort}"); } } if (!PlatformUtil.RunningOnWindows) { if (container.IsJobContainer) { // Ensure bash exist in the image int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"sh -c \"command -v bash\""); if (execWhichBashExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execWhichBashExitCode}"); } // Get current username container.CurrentUserName = (await ExecuteCommandAsync(executionContext, "whoami", string.Empty)).FirstOrDefault(); ArgUtil.NotNullOrEmpty(container.CurrentUserName, nameof(container.CurrentUserName)); // Get current userId container.CurrentUserId = (await ExecuteCommandAsync(executionContext, "id", $"-u {container.CurrentUserName}")).FirstOrDefault(); ArgUtil.NotNullOrEmpty(container.CurrentUserId, nameof(container.CurrentUserId)); executionContext.Output(StringUtil.Loc("CreateUserWithSameUIDInsideContainer", container.CurrentUserId)); // Create an user with same uid as the agent run as user inside the container. // All command execute in docker will run as Root by default, // this will cause the agent on the host machine doesn't have permission to any new file/folder created inside the container. // So, we create a user account with same UID inside the container and let all docker exec command run as that user. string containerUserName = string.Empty; // We need to find out whether there is a user with same UID inside the container List <string> userNames = new List <string>(); int execGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"grep {container.CurrentUserId} /etc/passwd | cut -f1 -d:\"", userNames); if (execGrepExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGrepExitCode}"); } if (userNames.Count > 0) { // check all potential username that might match the UID. foreach (string username in userNames) { int execIdExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"id -u {username}"); if (execIdExitCode == 0) { containerUserName = username; break; } } } // Create a new user with same UID if (string.IsNullOrEmpty(containerUserName)) { containerUserName = $"{container.CurrentUserName}_azpcontainer"; int execUseraddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"useradd -m -u {container.CurrentUserId} {containerUserName}"); if (execUseraddExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execUseraddExitCode}"); } } executionContext.Output(StringUtil.Loc("GrantContainerUserSUDOPrivilege", containerUserName)); // Create a new group for giving sudo permission int execGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd azure_pipelines_sudo"); if (execGroupaddExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGroupaddExitCode}"); } // Add the new created user to the new created sudo group. int execUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G azure_pipelines_sudo {containerUserName}"); if (execUsermodExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execUsermodExitCode}"); } // Allow the new sudo group run any sudo command without providing password. int execEchoExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"su -c \"echo '%azure_pipelines_sudo ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers\""); if (execUsermodExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execEchoExitCode}"); } bool setupDockerGroup = executionContext.Variables.GetBoolean("VSTS_SETUP_DOCKERGROUP") ?? StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("VSTS_SETUP_DOCKERGROUP"), true); if (setupDockerGroup) { executionContext.Output(StringUtil.Loc("AllowContainerUserRunDocker", containerUserName)); // Get docker.sock group id on Host string statFormatOption = "-c %g"; if (PlatformUtil.RunningOnMacOS) { statFormatOption = "-f %g"; } string dockerSockGroupId = (await ExecuteCommandAsync(executionContext, "stat", $"{statFormatOption} /var/run/docker.sock")).FirstOrDefault(); // We need to find out whether there is a group with same GID inside the container string existingGroupName = null; List <string> groupsOutput = new List <string>(); int execGroupGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"cat /etc/group\"", groupsOutput); if (execGroupGrepExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGroupGrepExitCode}"); } if (groupsOutput.Count > 0) { // check all potential groups that might match the GID. foreach (string groupOutput in groupsOutput) { if (!string.IsNullOrEmpty(groupOutput)) { var groupSegments = groupOutput.Split(':'); if (groupSegments.Length != 4) { Trace.Warning($"Unexpected output from /etc/group: '{groupOutput}'"); } else { // the output of /etc/group should looks like `group:x:gid:` var groupName = groupSegments[0]; var groupId = groupSegments[2]; if (string.Equals(dockerSockGroupId, groupId)) { existingGroupName = groupName; break; } } } } } if (string.IsNullOrEmpty(existingGroupName)) { // create a new group with same gid existingGroupName = "azure_pipelines_docker"; int execDockerGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd -g {dockerSockGroupId} azure_pipelines_docker"); if (execDockerGroupaddExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execDockerGroupaddExitCode}"); } } // Add the new created user to the docker socket group. int execGroupUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G {existingGroupName} {containerUserName}"); if (execGroupUsermodExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGroupUsermodExitCode}"); } // if path to node is just 'node', with no path, let's make sure it is actually there if (string.Equals(container.ContainerBringNodePath, "node", StringComparison.OrdinalIgnoreCase)) { List <string> nodeVersionOutput = new List <string>(); int execNodeVersionExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"node -v\"", nodeVersionOutput); if (execNodeVersionExitCode != 0) { throw new InvalidOperationException($"Unable to get node version on container {container.ContainerId}. Got exit code {execNodeVersionExitCode} from docker exec"); } if (nodeVersionOutput.Count > 0) { executionContext.Output($"Detected Node Version: {nodeVersionOutput[0]}"); Trace.Info($"Using node version {nodeVersionOutput[0]} in container {container.ContainerId}"); } else { throw new InvalidOperationException($"Unable to get node version on container {container.ContainerId}. No output from node -v"); } } } } } }
public async Task StartContainerAsync(IExecutionContext executionContext, object data) { Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); ContainerInfo container = data as ContainerInfo; ArgUtil.NotNull(container, nameof(container)); ArgUtil.NotNullOrEmpty(container.ContainerImage, nameof(container.ContainerImage)); Trace.Info($"Container name: {container.ContainerName}"); Trace.Info($"Container image: {container.ContainerImage}"); Trace.Info($"Container registry: {container.ContainerRegistryEndpoint.ToString()}"); Trace.Info($"Container options: {container.ContainerCreateOptions}"); Trace.Info($"Skip container image pull: {container.SkipContainerImagePull}"); // Check whether we are inside a container. // Our container feature requires to map working directory from host to the container. // If we are already inside a container, we will not able to find out the real working direcotry path on the host. #if OS_WINDOWS // service CExecSvc is Container Execution Agent. ServiceController[] scServices = ServiceController.GetServices(); if (scServices.Any(x => String.Equals(x.ServiceName, "cexecsvc", StringComparison.OrdinalIgnoreCase) && x.Status == ServiceControllerStatus.Running)) { throw new NotSupportedException(StringUtil.Loc("AgentAlreadyInsideContainer")); } #else var initProcessCgroup = File.ReadLines("/proc/1/cgroup"); if (initProcessCgroup.Any(x => x.IndexOf(":/docker/", StringComparison.OrdinalIgnoreCase) >= 0)) { throw new NotSupportedException(StringUtil.Loc("AgentAlreadyInsideContainer")); } #endif #if OS_WINDOWS // Check OS version (Windows server 1803 is required) object windowsInstallationType = Registry.GetValue(@"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion", "InstallationType", defaultValue: null); ArgUtil.NotNull(windowsInstallationType, nameof(windowsInstallationType)); object windowsReleaseId = Registry.GetValue(@"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion", "ReleaseId", defaultValue: null); ArgUtil.NotNull(windowsReleaseId, nameof(windowsReleaseId)); executionContext.Debug($"Current Windows version: '{windowsReleaseId} ({windowsInstallationType})'"); if (int.TryParse(windowsReleaseId.ToString(), out int releaseId)) { if (!windowsInstallationType.ToString().StartsWith("Server", StringComparison.OrdinalIgnoreCase) || releaseId < 1803) { throw new NotSupportedException(StringUtil.Loc("ContainerWindowsVersionRequirement")); } } else { throw new ArgumentOutOfRangeException(@"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\ReleaseId"); } #endif // Check docker client/server version DockerVersion dockerVersion = await _dockerManger.DockerVersion(executionContext); ArgUtil.NotNull(dockerVersion.ServerVersion, nameof(dockerVersion.ServerVersion)); ArgUtil.NotNull(dockerVersion.ClientVersion, nameof(dockerVersion.ClientVersion)); #if OS_WINDOWS Version requiredDockerVersion = new Version(17, 6); #else Version requiredDockerVersion = new Version(17, 12); #endif if (dockerVersion.ServerVersion < requiredDockerVersion) { throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerServerVersion", requiredDockerVersion, _dockerManger.DockerPath, dockerVersion.ServerVersion)); } if (dockerVersion.ClientVersion < requiredDockerVersion) { throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerClientVersion", requiredDockerVersion, _dockerManger.DockerPath, dockerVersion.ClientVersion)); } // Login to private docker registry string registryServer = string.Empty; if (container.ContainerRegistryEndpoint != Guid.Empty) { var registryEndpoint = executionContext.Endpoints.FirstOrDefault(x => x.Type == "dockerregistry" && x.Id == container.ContainerRegistryEndpoint); ArgUtil.NotNull(registryEndpoint, nameof(registryEndpoint)); string username = string.Empty; string password = string.Empty; registryEndpoint.Authorization?.Parameters?.TryGetValue("registry", out registryServer); registryEndpoint.Authorization?.Parameters?.TryGetValue("username", out username); registryEndpoint.Authorization?.Parameters?.TryGetValue("password", out password); ArgUtil.NotNullOrEmpty(registryServer, nameof(registryServer)); ArgUtil.NotNullOrEmpty(username, nameof(username)); ArgUtil.NotNullOrEmpty(password, nameof(password)); int loginExitCode = await _dockerManger.DockerLogin(executionContext, registryServer, username, password); if (loginExitCode != 0) { throw new InvalidOperationException($"Docker login fail with exit code {loginExitCode}"); } } try { if (!container.SkipContainerImagePull) { if (!string.IsNullOrEmpty(registryServer) && registryServer.IndexOf("index.docker.io", StringComparison.OrdinalIgnoreCase) < 0) { var registryServerUri = new Uri(registryServer); if (!container.ContainerImage.StartsWith(registryServerUri.Authority, StringComparison.OrdinalIgnoreCase)) { container.ContainerImage = $"{registryServerUri.Authority}/{container.ContainerImage}"; } } // Pull down docker image with retry up to 3 times int retryCount = 0; int pullExitCode = 0; while (retryCount < 3) { pullExitCode = await _dockerManger.DockerPull(executionContext, container.ContainerImage); if (pullExitCode == 0) { break; } else { retryCount++; if (retryCount < 3) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10)); executionContext.Warning($"Docker pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } } } if (retryCount == 3 && pullExitCode != 0) { throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}"); } } // Mount folder into container #if OS_WINDOWS container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Work), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Work)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools)))); #else string workingDirectory = Path.GetDirectoryName(executionContext.Variables.Get(Constants.Variables.System.DefaultWorkingDirectory).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)); container.MountVolumes.Add(new MountVolume(container.TranslateToHostPath(workingDirectory), workingDirectory)); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Temp), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Temp)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tasks), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tasks)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)), true)); // Ensure .taskkey file exist so we can mount it. string taskKeyFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), ".taskkey"); if (!File.Exists(taskKeyFile)) { File.WriteAllText(taskKeyFile, string.Empty); } container.MountVolumes.Add(new MountVolume(taskKeyFile, container.TranslateToContainerPath(taskKeyFile))); #endif #if !OS_WINDOWS if (string.IsNullOrEmpty(container.ContainerNetwork)) // create network when Windows support it. { // Create local docker network for this job to avoid port conflict when multiple agents run on same machine. container.ContainerNetwork = $"vsts_network_{Guid.NewGuid().ToString("N")}"; int networkExitCode = await _dockerManger.DockerNetworkCreate(executionContext, container.ContainerNetwork); if (networkExitCode != 0) { throw new InvalidOperationException($"Docker network create fail with exit code {networkExitCode}"); } // Expose docker network to env executionContext.Variables.Set(Constants.Variables.Agent.ContainerNetwork, container.ContainerNetwork); } #endif container.ContainerId = await _dockerManger.DockerCreate(context : executionContext, displayName : container.ContainerDisplayName, image : container.ContainerImage, mountVolumes : container.MountVolumes, network : container.ContainerNetwork, options : container.ContainerCreateOptions, environment : container.ContainerEnvironmentVariables); ArgUtil.NotNullOrEmpty(container.ContainerId, nameof(container.ContainerId)); executionContext.Variables.Set(Constants.Variables.Agent.ContainerId, container.ContainerId); // Start container int startExitCode = await _dockerManger.DockerStart(executionContext, container.ContainerId); if (startExitCode != 0) { throw new InvalidOperationException($"Docker start fail with exit code {startExitCode}"); } } finally { // Logout for private registry if (!string.IsNullOrEmpty(registryServer)) { int logoutExitCode = await _dockerManger.DockerLogout(executionContext, registryServer); if (logoutExitCode != 0) { executionContext.Error($"Docker logout fail with exit code {logoutExitCode}"); } } } #if !OS_WINDOWS // Ensure bash exist in the image int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"which bash"); if (execWhichBashExitCode != 0) { try { // Make sure container is up and running var psOutputs = await _dockerManger.DockerPS(executionContext, container.ContainerId, "--filter status=running"); if (psOutputs.FirstOrDefault(x => !string.IsNullOrEmpty(x))?.StartsWith(container.ContainerId) != true) { // container is not up and running, pull docker log for this container. await _dockerManger.DockerPS(executionContext, container.ContainerId, string.Empty); int logsExitCode = await _dockerManger.DockerLogs(executionContext, container.ContainerId); if (logsExitCode != 0) { executionContext.Warning($"Docker logs fail with exit code {logsExitCode}"); } executionContext.Warning($"Docker container {container.ContainerId} is not in running state."); } } catch (Exception ex) { // pull container log is best effort. Trace.Error("Catch exception when check container log and container status."); Trace.Error(ex); } throw new InvalidOperationException($"Docker exec fail with exit code {execWhichBashExitCode}"); } // Get current username container.CurrentUserName = (await ExecuteCommandAsync(executionContext, "whoami", string.Empty)).FirstOrDefault(); ArgUtil.NotNullOrEmpty(container.CurrentUserName, nameof(container.CurrentUserName)); // Get current userId container.CurrentUserId = (await ExecuteCommandAsync(executionContext, "id", $"-u {container.CurrentUserName}")).FirstOrDefault(); ArgUtil.NotNullOrEmpty(container.CurrentUserId, nameof(container.CurrentUserId)); executionContext.Output(StringUtil.Loc("CreateUserWithSameUIDInsideContainer", container.CurrentUserId)); // Create an user with same uid as the agent run as user inside the container. // All command execute in docker will run as Root by default, // this will cause the agent on the host machine doesn't have permission to any new file/folder created inside the container. // So, we create a user account with same UID inside the container and let all docker exec command run as that user. string containerUserName = string.Empty; // We need to find out whether there is a user with same UID inside the container List <string> userNames = new List <string>(); int execGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"grep {container.CurrentUserId} /etc/passwd | cut -f1 -d:\"", userNames); if (execGrepExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGrepExitCode}"); } if (userNames.Count > 0) { // check all potential username that might match the UID. foreach (string username in userNames) { int execIdExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"id -u {username}"); if (execIdExitCode == 0) { containerUserName = username; break; } } } // Create a new user with same UID if (string.IsNullOrEmpty(containerUserName)) { containerUserName = $"{container.CurrentUserName}_VSTSContainer"; int execUseraddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"useradd -m -u {container.CurrentUserId} {containerUserName}"); if (execUseraddExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execUseraddExitCode}"); } } executionContext.Output(StringUtil.Loc("GrantContainerUserSUDOPrivilege", containerUserName)); // Create a new vsts_sudo group for giving sudo permission int execGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd VSTS_Container_SUDO"); if (execGroupaddExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGroupaddExitCode}"); } // Add the new created user to the new created VSTS_SUDO group. int execUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G VSTS_Container_SUDO {containerUserName}"); if (execUsermodExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execUsermodExitCode}"); } // Allow the new vsts_sudo group run any sudo command without providing password. int execEchoExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"su -c \"echo '%VSTS_Container_SUDO ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers\""); if (execUsermodExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execEchoExitCode}"); } #endif }
private async Task StartContainerAsync(IExecutionContext executionContext) { Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNullOrEmpty(executionContext.Container.ContainerImage, nameof(executionContext.Container.ContainerImage)); // Check docker client/server version DockerVersion dockerVersion = await _dockerManger.DockerVersion(executionContext); ArgUtil.NotNull(dockerVersion.ServerVersion, nameof(dockerVersion.ServerVersion)); ArgUtil.NotNull(dockerVersion.ClientVersion, nameof(dockerVersion.ClientVersion)); Version requiredDockerVersion = new Version(17, 3); if (dockerVersion.ServerVersion < requiredDockerVersion) { throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerServerVersion", requiredDockerVersion, _dockerManger.DockerPath, dockerVersion.ServerVersion)); } if (dockerVersion.ClientVersion < requiredDockerVersion) { throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerClientVersion", requiredDockerVersion, _dockerManger.DockerPath, dockerVersion.ClientVersion)); } // Pull down docker image int pullExitCode = await _dockerManger.DockerPull(executionContext, executionContext.Container.ContainerImage); if (pullExitCode != 0) { throw new InvalidOperationException($"Docker pull fail with exit code {pullExitCode}"); } // Mount folder into container executionContext.Container.MountVolumes.Add(new MountVolume(Path.GetDirectoryName(executionContext.Variables.System_DefaultWorkingDirectory.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)))); executionContext.Container.MountVolumes.Add(new MountVolume(executionContext.Variables.Agent_TempDirectory)); executionContext.Container.MountVolumes.Add(new MountVolume(executionContext.Variables.Agent_ToolsDirectory)); executionContext.Container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), true)); executionContext.Container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tasks), true)); // Ensure .taskkey file exist so we can mount it. string taskKeyFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), ".taskkey"); if (!File.Exists(taskKeyFile)) { File.WriteAllText(taskKeyFile, string.Empty); } executionContext.Container.MountVolumes.Add(new MountVolume(taskKeyFile)); executionContext.Container.ContainerId = await _dockerManger.DockerCreate(executionContext, executionContext.Container.ContainerImage, executionContext.Container.MountVolumes); ArgUtil.NotNullOrEmpty(executionContext.Container.ContainerId, nameof(executionContext.Container.ContainerId)); // Start container int startExitCode = await _dockerManger.DockerStart(executionContext, executionContext.Container.ContainerId); if (startExitCode != 0) { throw new InvalidOperationException($"Docker start fail with exit code {startExitCode}"); } // Ensure bash exist in the image int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, executionContext.Container.ContainerId, string.Empty, $"which bash"); if (execWhichBashExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execWhichBashExitCode}"); } // Get current username executionContext.Container.CurrentUserName = (await ExecuteCommandAsync(executionContext, "whoami", string.Empty)).FirstOrDefault(); ArgUtil.NotNullOrEmpty(executionContext.Container.CurrentUserName, nameof(executionContext.Container.CurrentUserName)); // Get current userId executionContext.Container.CurrentUserId = (await ExecuteCommandAsync(executionContext, "id", $"-u {executionContext.Container.CurrentUserName}")).FirstOrDefault(); ArgUtil.NotNullOrEmpty(executionContext.Container.CurrentUserId, nameof(executionContext.Container.CurrentUserId)); executionContext.Output(StringUtil.Loc("CreateUserWithSameUIDInsideContainer", executionContext.Container.CurrentUserId)); // Create an user with same uid as the agent run as user inside the container. // All command execute in docker will run as Root by default, // this will cause the agent on the host machine doesn't have permission to any new file/folder created inside the container. // So, we create a user account with same UID inside the container and let all docker exec command run as that user. string containerUserName = string.Empty; // We need to find out whether there is a user with same UID inside the container List <string> userNames = new List <string>(); int execGrepExitCode = await _dockerManger.DockerExec(executionContext, executionContext.Container.ContainerId, string.Empty, $"bash -c \"grep {executionContext.Container.CurrentUserId} /etc/passwd | cut -f1 -d:\"", userNames); if (execGrepExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGrepExitCode}"); } if (userNames.Count > 0) { // check all potential username that might match the UID. foreach (string username in userNames) { int execIdExitCode = await _dockerManger.DockerExec(executionContext, executionContext.Container.ContainerId, string.Empty, $"id -u {username}"); if (execIdExitCode == 0) { containerUserName = username; break; } } } // Create a new user with same UID if (string.IsNullOrEmpty(containerUserName)) { containerUserName = $"{executionContext.Container.CurrentUserName}_VSTSContainer"; int execUseraddExitCode = await _dockerManger.DockerExec(executionContext, executionContext.Container.ContainerId, string.Empty, $"useradd -m -u {executionContext.Container.CurrentUserId} {containerUserName}"); if (execUseraddExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execUseraddExitCode}"); } } executionContext.Output(StringUtil.Loc("GrantContainerUserSUDOPrivilege", containerUserName)); // Create a new vsts_sudo group for giving sudo permission int execGroupaddExitCode = await _dockerManger.DockerExec(executionContext, executionContext.Container.ContainerId, string.Empty, $"groupadd VSTS_Container_SUDO"); if (execGroupaddExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGroupaddExitCode}"); } // Add the new created user to the new created VSTS_SUDO group. int execUsermodExitCode = await _dockerManger.DockerExec(executionContext, executionContext.Container.ContainerId, string.Empty, $"usermod -a -G VSTS_Container_SUDO {containerUserName}"); if (execUsermodExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execUsermodExitCode}"); } // Allow the new vsts_sudo group run any sudo command without providing password. int execEchoExitCode = await _dockerManger.DockerExec(executionContext, executionContext.Container.ContainerId, string.Empty, $"su -c \"echo '%VSTS_Container_SUDO ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers\""); if (execUsermodExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execEchoExitCode}"); } }
private async Task StartContainerAsync(IExecutionContext executionContext) { Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNullOrEmpty(executionContext.Container.ContainerImage, nameof(executionContext.Container.ContainerImage)); // Check docker client/server version DockerVersion dockerVersion = await _dockerManger.DockerVersion(executionContext); ArgUtil.NotNull(dockerVersion.ServerVersion, nameof(dockerVersion.ServerVersion)); ArgUtil.NotNull(dockerVersion.ClientVersion, nameof(dockerVersion.ClientVersion)); Version requiredDockerVersion = new Version(17, 3); if (dockerVersion.ServerVersion < requiredDockerVersion) { throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerServerVersion", requiredDockerVersion, _dockerManger.DockerPath, dockerVersion.ServerVersion)); } if (dockerVersion.ClientVersion < requiredDockerVersion) { throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerClientVersion", requiredDockerVersion, _dockerManger.DockerPath, dockerVersion.ClientVersion)); } // Pull down docker image int pullExitCode = await _dockerManger.DockerPull(executionContext, executionContext.Container.ContainerImage); if (pullExitCode != 0) { throw new InvalidOperationException($"Docker pull fail with exit code {pullExitCode}"); } // Mount folder into container executionContext.Container.MountVolumes.Add(new MountVolume(executionContext.Variables.System_DefaultWorkingDirectory)); executionContext.Container.MountVolumes.Add(new MountVolume(executionContext.Variables.Agent_TempDirectory)); executionContext.Container.MountVolumes.Add(new MountVolume(executionContext.Variables.Agent_ToolsDirectory)); executionContext.Container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), true)); executionContext.Container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tasks), true)); // Ensure .taskkey file exist so we can mount it. string taskKeyFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), ".taskkey"); if (!File.Exists(taskKeyFile)) { File.WriteAllText(taskKeyFile, string.Empty); } executionContext.Container.MountVolumes.Add(new MountVolume(taskKeyFile)); executionContext.Container.ContainerId = await _dockerManger.DockerCreate(executionContext, executionContext.Container.ContainerImage, executionContext.Container.MountVolumes); ArgUtil.NotNullOrEmpty(executionContext.Container.ContainerId, nameof(executionContext.Container.ContainerId)); // Get current username executionContext.Container.CurrentUserName = (await ExecuteCommandAsync(executionContext, "whoami", string.Empty)).FirstOrDefault(); ArgUtil.NotNullOrEmpty(executionContext.Container.CurrentUserName, nameof(executionContext.Container.CurrentUserName)); // Get current userId executionContext.Container.CurrentUserId = (await ExecuteCommandAsync(executionContext, "id", $"-u {executionContext.Container.CurrentUserName}")).FirstOrDefault(); ArgUtil.NotNullOrEmpty(executionContext.Container.CurrentUserId, nameof(executionContext.Container.CurrentUserId)); int startExitCode = await _dockerManger.DockerStart(executionContext, executionContext.Container.ContainerId); if (startExitCode != 0) { throw new InvalidOperationException($"Docker start fail with exit code {startExitCode}"); } // Ensure bash exist in the image int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, executionContext.Container.ContainerId, string.Empty, $"which bash"); if (execWhichBashExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execWhichBashExitCode}"); } // Create an user with same uid as the agent run as user inside the container. // All command execute in docker will run as Root by default, // this will cause the agent on the host machine doesn't have permission to any new file/folder created inside the container. // So, we create a user account with same UID inside the container and let all docker exec command run as that user. int execUseraddExitCode = await _dockerManger.DockerExec(executionContext, executionContext.Container.ContainerId, string.Empty, $"useradd -m -u {executionContext.Container.CurrentUserId} {executionContext.Container.CurrentUserName}_VSTSContainer"); if (execUseraddExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execUseraddExitCode}"); } }
public async Task StartContainerAsync(IExecutionContext executionContext, object data) { Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); ContainerInfo container = data as ContainerInfo; ArgUtil.NotNull(container, nameof(container)); ArgUtil.NotNullOrEmpty(container.ContainerImage, nameof(container.ContainerImage)); Trace.Info($"Container name: {container.ContainerName}"); Trace.Info($"Container image: {container.ContainerImage}"); Trace.Info($"Container registry: {container.ContainerRegistryEndpoint}"); Trace.Info($"Container options: {container.ContainerCreateOptions}"); Trace.Info($"Skip container image pull: {container.SkipContainerImagePull}"); // Check docker client/server version DockerVersion dockerVersion = await _dockerManger.DockerVersion(executionContext); ArgUtil.NotNull(dockerVersion.ServerVersion, nameof(dockerVersion.ServerVersion)); ArgUtil.NotNull(dockerVersion.ClientVersion, nameof(dockerVersion.ClientVersion)); #if OS_WINDOWS Version requiredDockerVersion = new Version(17, 6); #else Version requiredDockerVersion = new Version(17, 12); #endif if (dockerVersion.ServerVersion < requiredDockerVersion) { throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerServerVersion", requiredDockerVersion, _dockerManger.DockerPath, dockerVersion.ServerVersion)); } if (dockerVersion.ClientVersion < requiredDockerVersion) { throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerClientVersion", requiredDockerVersion, _dockerManger.DockerPath, dockerVersion.ClientVersion)); } // Login to private docker registry string registryServer = string.Empty; if (!string.IsNullOrEmpty(container.ContainerRegistryEndpoint)) { var registryEndpoint = executionContext.Endpoints.FirstOrDefault(x => x.Type == "dockerregistry" && String.Equals(x.Id.ToString(), container.ContainerRegistryEndpoint, StringComparison.OrdinalIgnoreCase)); ArgUtil.NotNull(registryEndpoint, nameof(registryEndpoint)); string username = string.Empty; string password = string.Empty; registryEndpoint.Authorization?.Parameters?.TryGetValue("registry", out registryServer); registryEndpoint.Authorization?.Parameters?.TryGetValue("username", out username); registryEndpoint.Authorization?.Parameters?.TryGetValue("password", out password); ArgUtil.NotNullOrEmpty(registryServer, nameof(registryServer)); ArgUtil.NotNullOrEmpty(username, nameof(username)); ArgUtil.NotNullOrEmpty(password, nameof(password)); int loginExitCode = await _dockerManger.DockerLogin(executionContext, registryServer, username, password); if (loginExitCode != 0) { throw new InvalidOperationException($"Docker login fail with exit code {loginExitCode}"); } } try { if (!container.SkipContainerImagePull) { string imageName = container.ContainerImage; if (!string.IsNullOrEmpty(registryServer) && registryServer.IndexOf("index.docker.io", StringComparison.OrdinalIgnoreCase) < 0 && !imageName.StartsWith(registryServer, StringComparison.OrdinalIgnoreCase)) { imageName = $"{registryServer}/{imageName}"; } // Pull down docker image int pullExitCode = await _dockerManger.DockerPull(executionContext, imageName); if (pullExitCode != 0) { throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}"); } } // Mount folder into container #if OS_WINDOWS container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), "C:\\_a\\externals")); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Work), "C:\\_work")); container.MountVolumes.Add(new MountVolume(executionContext.Variables.Agent_ToolsDirectory, "C:\\_work\\_tool")); container.PathMappings[HostContext.GetDirectory(WellKnownDirectory.Externals)] = "C:\\_a\\externals"; container.PathMappings[HostContext.GetDirectory(WellKnownDirectory.Work)] = "C:\\_work"; container.PathMappings[executionContext.Variables.Agent_ToolsDirectory] = "C:\\_work\\_tool"; #else string workingDirMountSource = Path.GetDirectoryName(executionContext.Variables.System_DefaultWorkingDirectory.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)); string workingDirMountTarget = workingDirMountSource.Replace(HostContext.GetDirectory(WellKnownDirectory.Work), "/_work"); container.MountVolumes.Add(new MountVolume(workingDirMountSource, workingDirMountTarget)); container.MountVolumes.Add(new MountVolume(executionContext.Variables.Agent_TempDirectory, "/_work/_temp")); container.MountVolumes.Add(new MountVolume(executionContext.Variables.Agent_ToolsDirectory, "/_work/_tool")); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tasks), "/_work/_tasks")); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), "/_a/externals", true)); // Ensure .taskkey file exist so we can mount it. string taskKeyFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), ".taskkey"); if (!File.Exists(taskKeyFile)) { File.WriteAllText(taskKeyFile, string.Empty); } container.MountVolumes.Add(new MountVolume(taskKeyFile, "/_work/.taskkey")); container.PathMappings[HostContext.GetDirectory(WellKnownDirectory.Externals)] = "/_a/externals"; container.PathMappings[HostContext.GetDirectory(WellKnownDirectory.Work)] = "/_work"; container.PathMappings[executionContext.Variables.Agent_ToolsDirectory] = "/_work/_tool"; #endif #if !OS_WINDOWS if (string.IsNullOrEmpty(container.ContainerNetwork)) // create network when Windows support it. { // Create local docker network for this job to avoid port conflict when multiple agents run on same machine. container.ContainerNetwork = $"vsts_network_{Guid.NewGuid().ToString("N")}"; int networkExitCode = await _dockerManger.DockerNetworkCreate(executionContext, container.ContainerNetwork); if (networkExitCode != 0) { throw new InvalidOperationException($"Docker network create fail with exit code {networkExitCode}"); } // Expose docker network to env executionContext.Variables.Set(Constants.Variables.Agent.ContainerNetwork, container.ContainerNetwork); } #endif container.ContainerId = await _dockerManger.DockerCreate(context : executionContext, displayName : container.ContainerDisplayName, image : container.ContainerImage, mountVolumes : container.MountVolumes, network : container.ContainerNetwork, options : container.ContainerCreateOptions); ArgUtil.NotNullOrEmpty(container.ContainerId, nameof(container.ContainerId)); executionContext.Variables.Set(Constants.Variables.Agent.ContainerId, container.ContainerId); // Start container int startExitCode = await _dockerManger.DockerStart(executionContext, container.ContainerId); if (startExitCode != 0) { throw new InvalidOperationException($"Docker start fail with exit code {startExitCode}"); } } finally { // Logout for private registry if (!string.IsNullOrEmpty(registryServer)) { int logoutExitCode = await _dockerManger.DockerLogout(executionContext, registryServer); if (logoutExitCode != 0) { executionContext.Error($"Docker logout fail with exit code {logoutExitCode}"); } } } #if !OS_WINDOWS // Ensure bash exist in the image int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"which bash"); if (execWhichBashExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execWhichBashExitCode}"); } // Get current username container.CurrentUserName = (await ExecuteCommandAsync(executionContext, "whoami", string.Empty)).FirstOrDefault(); ArgUtil.NotNullOrEmpty(container.CurrentUserName, nameof(container.CurrentUserName)); // Get current userId container.CurrentUserId = (await ExecuteCommandAsync(executionContext, "id", $"-u {container.CurrentUserName}")).FirstOrDefault(); ArgUtil.NotNullOrEmpty(container.CurrentUserId, nameof(container.CurrentUserId)); executionContext.Output(StringUtil.Loc("CreateUserWithSameUIDInsideContainer", container.CurrentUserId)); // Create an user with same uid as the agent run as user inside the container. // All command execute in docker will run as Root by default, // this will cause the agent on the host machine doesn't have permission to any new file/folder created inside the container. // So, we create a user account with same UID inside the container and let all docker exec command run as that user. string containerUserName = string.Empty; // We need to find out whether there is a user with same UID inside the container List <string> userNames = new List <string>(); int execGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"grep {container.CurrentUserId} /etc/passwd | cut -f1 -d:\"", userNames); if (execGrepExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGrepExitCode}"); } if (userNames.Count > 0) { // check all potential username that might match the UID. foreach (string username in userNames) { int execIdExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"id -u {username}"); if (execIdExitCode == 0) { containerUserName = username; break; } } } // Create a new user with same UID if (string.IsNullOrEmpty(containerUserName)) { containerUserName = $"{container.CurrentUserName}_VSTSContainer"; int execUseraddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"useradd -m -u {container.CurrentUserId} {containerUserName}"); if (execUseraddExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execUseraddExitCode}"); } } executionContext.Output(StringUtil.Loc("GrantContainerUserSUDOPrivilege", containerUserName)); // Create a new vsts_sudo group for giving sudo permission int execGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd VSTS_Container_SUDO"); if (execGroupaddExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGroupaddExitCode}"); } // Add the new created user to the new created VSTS_SUDO group. int execUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G VSTS_Container_SUDO {containerUserName}"); if (execUsermodExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execUsermodExitCode}"); } // Allow the new vsts_sudo group run any sudo command without providing password. int execEchoExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"su -c \"echo '%VSTS_Container_SUDO ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers\""); if (execUsermodExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execEchoExitCode}"); } #endif }