public async Task StartContainerAsync(IExecutionContext executionContext, object data) { Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); ContainerInfo container = data as ContainerInfo; ArgUtil.NotNull(container, nameof(container)); ArgUtil.NotNullOrEmpty(container.ContainerImage, nameof(container.ContainerImage)); Trace.Info($"Container name: {container.ContainerName}"); Trace.Info($"Container image: {container.ContainerImage}"); Trace.Info($"Container registry: {container.ContainerRegistryEndpoint.ToString()}"); Trace.Info($"Container options: {container.ContainerCreateOptions}"); Trace.Info($"Skip container image pull: {container.SkipContainerImagePull}"); // Check whether we are inside a container. // Our container feature requires to map working directory from host to the container. // If we are already inside a container, we will not able to find out the real working direcotry path on the host. #if OS_WINDOWS // service CExecSvc is Container Execution Agent. ServiceController[] scServices = ServiceController.GetServices(); if (scServices.Any(x => String.Equals(x.ServiceName, "cexecsvc", StringComparison.OrdinalIgnoreCase) && x.Status == ServiceControllerStatus.Running)) { throw new NotSupportedException(StringUtil.Loc("AgentAlreadyInsideContainer")); } #else var initProcessCgroup = File.ReadLines("/proc/1/cgroup"); if (initProcessCgroup.Any(x => x.IndexOf(":/docker/", StringComparison.OrdinalIgnoreCase) >= 0)) { throw new NotSupportedException(StringUtil.Loc("AgentAlreadyInsideContainer")); } #endif #if OS_WINDOWS // Check OS version (Windows server 1803 is required) object windowsInstallationType = Registry.GetValue(@"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion", "InstallationType", defaultValue: null); ArgUtil.NotNull(windowsInstallationType, nameof(windowsInstallationType)); object windowsReleaseId = Registry.GetValue(@"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion", "ReleaseId", defaultValue: null); ArgUtil.NotNull(windowsReleaseId, nameof(windowsReleaseId)); executionContext.Debug($"Current Windows version: '{windowsReleaseId} ({windowsInstallationType})'"); if (int.TryParse(windowsReleaseId.ToString(), out int releaseId)) { if (!windowsInstallationType.ToString().StartsWith("Server", StringComparison.OrdinalIgnoreCase) || releaseId < 1803) { throw new NotSupportedException(StringUtil.Loc("ContainerWindowsVersionRequirement")); } } else { throw new ArgumentOutOfRangeException(@"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\ReleaseId"); } #endif // Check docker client/server version DockerVersion dockerVersion = await _dockerManger.DockerVersion(executionContext); ArgUtil.NotNull(dockerVersion.ServerVersion, nameof(dockerVersion.ServerVersion)); ArgUtil.NotNull(dockerVersion.ClientVersion, nameof(dockerVersion.ClientVersion)); #if OS_WINDOWS Version requiredDockerVersion = new Version(17, 6); #else Version requiredDockerVersion = new Version(17, 12); #endif if (dockerVersion.ServerVersion < requiredDockerVersion) { throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerServerVersion", requiredDockerVersion, _dockerManger.DockerPath, dockerVersion.ServerVersion)); } if (dockerVersion.ClientVersion < requiredDockerVersion) { throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerClientVersion", requiredDockerVersion, _dockerManger.DockerPath, dockerVersion.ClientVersion)); } // Login to private docker registry string registryServer = string.Empty; if (container.ContainerRegistryEndpoint != Guid.Empty) { var registryEndpoint = executionContext.Endpoints.FirstOrDefault(x => x.Type == "dockerregistry" && x.Id == container.ContainerRegistryEndpoint); ArgUtil.NotNull(registryEndpoint, nameof(registryEndpoint)); string username = string.Empty; string password = string.Empty; registryEndpoint.Authorization?.Parameters?.TryGetValue("registry", out registryServer); registryEndpoint.Authorization?.Parameters?.TryGetValue("username", out username); registryEndpoint.Authorization?.Parameters?.TryGetValue("password", out password); ArgUtil.NotNullOrEmpty(registryServer, nameof(registryServer)); ArgUtil.NotNullOrEmpty(username, nameof(username)); ArgUtil.NotNullOrEmpty(password, nameof(password)); int loginExitCode = await _dockerManger.DockerLogin(executionContext, registryServer, username, password); if (loginExitCode != 0) { throw new InvalidOperationException($"Docker login fail with exit code {loginExitCode}"); } } try { if (!container.SkipContainerImagePull) { string imageName = container.ContainerImage; if (!string.IsNullOrEmpty(registryServer) && registryServer.IndexOf("index.docker.io", StringComparison.OrdinalIgnoreCase) < 0) { var registryServerUri = new Uri(registryServer); if (!imageName.StartsWith(registryServerUri.Authority, StringComparison.OrdinalIgnoreCase)) { imageName = $"{registryServerUri.Authority}/{imageName}"; } } // Pull down docker image with retry up to 3 times int retryCount = 0; int pullExitCode = 0; while (retryCount < 3) { pullExitCode = await _dockerManger.DockerPull(executionContext, imageName); if (pullExitCode == 0) { break; } else { retryCount++; if (retryCount < 3) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10)); executionContext.Warning($"Docker pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } } } if (retryCount == 3 && pullExitCode != 0) { throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}"); } } // Mount folder into container #if OS_WINDOWS container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Work), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Work)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools)))); #else string workingDirectory = Path.GetDirectoryName(executionContext.Variables.Get(Constants.Variables.System.DefaultWorkingDirectory).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)); container.MountVolumes.Add(new MountVolume(container.TranslateToHostPath(workingDirectory), workingDirectory)); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Temp), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Temp)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tasks), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tasks)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)), true)); // Ensure .taskkey file exist so we can mount it. string taskKeyFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), ".taskkey"); if (!File.Exists(taskKeyFile)) { File.WriteAllText(taskKeyFile, string.Empty); } container.MountVolumes.Add(new MountVolume(taskKeyFile, container.TranslateToContainerPath(taskKeyFile))); #endif #if !OS_WINDOWS if (string.IsNullOrEmpty(container.ContainerNetwork)) // create network when Windows support it. { // Create local docker network for this job to avoid port conflict when multiple agents run on same machine. container.ContainerNetwork = $"vsts_network_{Guid.NewGuid().ToString("N")}"; int networkExitCode = await _dockerManger.DockerNetworkCreate(executionContext, container.ContainerNetwork); if (networkExitCode != 0) { throw new InvalidOperationException($"Docker network create fail with exit code {networkExitCode}"); } // Expose docker network to env executionContext.Variables.Set(Constants.Variables.Agent.ContainerNetwork, container.ContainerNetwork); } #endif container.ContainerId = await _dockerManger.DockerCreate(context : executionContext, displayName : container.ContainerDisplayName, image : container.ContainerImage, mountVolumes : container.MountVolumes, network : container.ContainerNetwork, options : container.ContainerCreateOptions, environment : container.ContainerEnvironmentVariables); ArgUtil.NotNullOrEmpty(container.ContainerId, nameof(container.ContainerId)); executionContext.Variables.Set(Constants.Variables.Agent.ContainerId, container.ContainerId); // Start container int startExitCode = await _dockerManger.DockerStart(executionContext, container.ContainerId); if (startExitCode != 0) { throw new InvalidOperationException($"Docker start fail with exit code {startExitCode}"); } } finally { // Logout for private registry if (!string.IsNullOrEmpty(registryServer)) { int logoutExitCode = await _dockerManger.DockerLogout(executionContext, registryServer); if (logoutExitCode != 0) { executionContext.Error($"Docker logout fail with exit code {logoutExitCode}"); } } } #if !OS_WINDOWS // Ensure bash exist in the image int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"which bash"); if (execWhichBashExitCode != 0) { try { // Make sure container is up and running var psOutputs = await _dockerManger.DockerPS(executionContext, container.ContainerId, "--filter status=running"); if (psOutputs.FirstOrDefault(x => !string.IsNullOrEmpty(x))?.StartsWith(container.ContainerId) != true) { // container is not up and running, pull docker log for this container. await _dockerManger.DockerPS(executionContext, container.ContainerId, string.Empty); int logsExitCode = await _dockerManger.DockerLogs(executionContext, container.ContainerId); if (logsExitCode != 0) { executionContext.Warning($"Docker logs fail with exit code {logsExitCode}"); } executionContext.Warning($"Docker container {container.ContainerId} is not in running state."); } } catch (Exception ex) { // pull container log is best effort. Trace.Error("Catch exception when check container log and container status."); Trace.Error(ex); } throw new InvalidOperationException($"Docker exec fail with exit code {execWhichBashExitCode}"); } // Get current username container.CurrentUserName = (await ExecuteCommandAsync(executionContext, "whoami", string.Empty)).FirstOrDefault(); ArgUtil.NotNullOrEmpty(container.CurrentUserName, nameof(container.CurrentUserName)); // Get current userId container.CurrentUserId = (await ExecuteCommandAsync(executionContext, "id", $"-u {container.CurrentUserName}")).FirstOrDefault(); ArgUtil.NotNullOrEmpty(container.CurrentUserId, nameof(container.CurrentUserId)); executionContext.Output(StringUtil.Loc("CreateUserWithSameUIDInsideContainer", container.CurrentUserId)); // Create an user with same uid as the agent run as user inside the container. // All command execute in docker will run as Root by default, // this will cause the agent on the host machine doesn't have permission to any new file/folder created inside the container. // So, we create a user account with same UID inside the container and let all docker exec command run as that user. string containerUserName = string.Empty; // We need to find out whether there is a user with same UID inside the container List <string> userNames = new List <string>(); int execGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"grep {container.CurrentUserId} /etc/passwd | cut -f1 -d:\"", userNames); if (execGrepExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGrepExitCode}"); } if (userNames.Count > 0) { // check all potential username that might match the UID. foreach (string username in userNames) { int execIdExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"id -u {username}"); if (execIdExitCode == 0) { containerUserName = username; break; } } } // Create a new user with same UID if (string.IsNullOrEmpty(containerUserName)) { containerUserName = $"{container.CurrentUserName}_VSTSContainer"; int execUseraddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"useradd -m -u {container.CurrentUserId} {containerUserName}"); if (execUseraddExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execUseraddExitCode}"); } } executionContext.Output(StringUtil.Loc("GrantContainerUserSUDOPrivilege", containerUserName)); // Create a new vsts_sudo group for giving sudo permission int execGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd VSTS_Container_SUDO"); if (execGroupaddExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGroupaddExitCode}"); } // Add the new created user to the new created VSTS_SUDO group. int execUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G VSTS_Container_SUDO {containerUserName}"); if (execUsermodExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execUsermodExitCode}"); } // Allow the new vsts_sudo group run any sudo command without providing password. int execEchoExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"su -c \"echo '%VSTS_Container_SUDO ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers\""); if (execUsermodExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execEchoExitCode}"); } #endif }
private async Task PullContainerAsync(IExecutionContext executionContext, ContainerInfo container) { Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(container, nameof(container)); ArgUtil.NotNullOrEmpty(container.ContainerImage, nameof(container.ContainerImage)); Trace.Info($"Container name: {container.ContainerName}"); Trace.Info($"Container image: {container.ContainerImage}"); Trace.Info($"Container registry: {container.ContainerRegistryEndpoint.ToString()}"); Trace.Info($"Container options: {container.ContainerCreateOptions}"); Trace.Info($"Skip container image pull: {container.SkipContainerImagePull}"); // Login to private docker registry string registryServer = string.Empty; if (container.ContainerRegistryEndpoint != Guid.Empty) { var registryEndpoint = executionContext.Endpoints.FirstOrDefault(x => x.Type == "dockerregistry" && x.Id == container.ContainerRegistryEndpoint); ArgUtil.NotNull(registryEndpoint, nameof(registryEndpoint)); string username = string.Empty; string password = string.Empty; string registryType = string.Empty; registryEndpoint.Data?.TryGetValue("registrytype", out registryType); if (string.Equals(registryType, "ACR", StringComparison.OrdinalIgnoreCase)) { string loginServer = string.Empty; registryEndpoint.Authorization?.Parameters?.TryGetValue("loginServer", out loginServer); if (loginServer != null) { loginServer = loginServer.ToLower(); } registryEndpoint.Authorization?.Parameters?.TryGetValue("serviceprincipalid", out username); registryEndpoint.Authorization?.Parameters?.TryGetValue("serviceprincipalkey", out password); registryServer = $"https://{loginServer}"; } else { registryEndpoint.Authorization?.Parameters?.TryGetValue("registry", out registryServer); registryEndpoint.Authorization?.Parameters?.TryGetValue("username", out username); registryEndpoint.Authorization?.Parameters?.TryGetValue("password", out password); } ArgUtil.NotNullOrEmpty(registryServer, nameof(registryServer)); ArgUtil.NotNullOrEmpty(username, nameof(username)); ArgUtil.NotNullOrEmpty(password, nameof(password)); int loginExitCode = await _dockerManger.DockerLogin(executionContext, registryServer, username, password); if (loginExitCode != 0) { throw new InvalidOperationException($"Docker login fail with exit code {loginExitCode}"); } } try { if (!container.SkipContainerImagePull) { if (!string.IsNullOrEmpty(registryServer) && registryServer.IndexOf("index.docker.io", StringComparison.OrdinalIgnoreCase) < 0) { var registryServerUri = new Uri(registryServer); if (!container.ContainerImage.StartsWith(registryServerUri.Authority, StringComparison.OrdinalIgnoreCase)) { container.ContainerImage = $"{registryServerUri.Authority}/{container.ContainerImage}"; } } // Pull down docker image with retry up to 3 times int retryCount = 0; int pullExitCode = 0; while (retryCount < 3) { pullExitCode = await _dockerManger.DockerPull(executionContext, container.ContainerImage); if (pullExitCode == 0) { break; } else { retryCount++; if (retryCount < 3) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10)); executionContext.Warning($"Docker pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } } } if (retryCount == 3 && pullExitCode != 0) { throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}"); } } if (PlatformUtil.RunningOnMacOS) { container.ImageOS = PlatformUtil.OS.Linux; } // if running on Windows, and attempting to run linux container, require container to have node else if (PlatformUtil.RunningOnWindows) { string containerOS = await _dockerManger.DockerInspect(context : executionContext, dockerObject : container.ContainerImage, options : $"--format=\"{{{{.Os}}}}\""); if (string.Equals("linux", containerOS, StringComparison.OrdinalIgnoreCase)) { container.ImageOS = PlatformUtil.OS.Linux; } } } finally { // Logout for private registry if (!string.IsNullOrEmpty(registryServer)) { int logoutExitCode = await _dockerManger.DockerLogout(executionContext, registryServer); if (logoutExitCode != 0) { executionContext.Error($"Docker logout fail with exit code {logoutExitCode}"); } } } }
private async Task DownloadAsync(IExecutionContext executionContext, Pipelines.TaskStepDefinitionReference task) { Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(task, nameof(task)); ArgUtil.NotNullOrEmpty(task.Version, nameof(task.Version)); var taskServer = HostContext.GetService <ITaskServer>(); // first check to see if we already have the task string destDirectory = GetDirectory(task); Trace.Info($"Ensuring task exists: ID '{task.Id}', version '{task.Version}', name '{task.Name}', directory '{destDirectory}'."); if (File.Exists(destDirectory + ".completed")) { executionContext.Debug($"Task '{task.Name}' already downloaded at '{destDirectory}'."); return; } // delete existing task folder. Trace.Verbose("Deleting task destination folder: {0}", destDirectory); IOUtil.DeleteDirectory(destDirectory, CancellationToken.None); // Inform the user that a download is taking place. The download could take a while if // the task zip is large. It would be nice to print the localized name, but it is not // available from the reference included in the job message. executionContext.Output(StringUtil.Loc("DownloadingTask0", task.Name)); string zipFile = string.Empty; var version = new TaskVersion(task.Version); //download and extract task in a temp folder and rename it on success string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Tasks), "_temp_" + Guid.NewGuid()); try { Directory.CreateDirectory(tempDirectory); int retryCount = 0; // Allow up to 20 * 60s for any task to be downloaded from service. // Base on Kusto, the longest we have on the service today is over 850 seconds. // Timeout limit can be overwrite by environment variable if (!int.TryParse(Environment.GetEnvironmentVariable("VSTS_TASK_DOWNLOAD_TIMEOUT") ?? string.Empty, out int timeoutSeconds)) { timeoutSeconds = 20 * 60; } while (retryCount < 3) { using (var taskDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds))) using (var taskDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(taskDownloadTimeout.Token, executionContext.CancellationToken)) { try { zipFile = Path.Combine(tempDirectory, string.Format("{0}.zip", Guid.NewGuid())); //open zip stream in async mode using (FileStream fs = new FileStream(zipFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true)) using (Stream result = await taskServer.GetTaskContentZipAsync(task.Id, version, taskDownloadCancellation.Token)) { await result.CopyToAsync(fs, _defaultCopyBufferSize, taskDownloadCancellation.Token); await fs.FlushAsync(taskDownloadCancellation.Token); // download succeed, break out the retry loop. break; } } catch (OperationCanceledException) when(executionContext.CancellationToken.IsCancellationRequested) { Trace.Info($"Task download has been cancelled."); throw; } catch (Exception ex) when(retryCount < 2) { retryCount++; Trace.Error($"Fail to download task '{task.Id} ({task.Name}/{task.Version})' -- Attempt: {retryCount}"); Trace.Error(ex); if (taskDownloadTimeout.Token.IsCancellationRequested) { // task download didn't finish within timeout executionContext.Warning(StringUtil.Loc("TaskDownloadTimeout", task.Name, timeoutSeconds)); } else { executionContext.Warning(StringUtil.Loc("TaskDownloadFailed", task.Name, ex.Message)); } } } if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("VSTS_TASK_DOWNLOAD_NO_BACKOFF"))) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30)); executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } } Directory.CreateDirectory(destDirectory); ZipFile.ExtractToDirectory(zipFile, destDirectory); Trace.Verbose("Create watermark file indicate task download succeed."); File.WriteAllText(destDirectory + ".completed", DateTime.UtcNow.ToString()); executionContext.Debug($"Task '{task.Name}' has been downloaded into '{destDirectory}'."); Trace.Info("Finished getting task."); } finally { try { //if the temp folder wasn't moved -> wipe it if (Directory.Exists(tempDirectory)) { Trace.Verbose("Deleting task temp folder: {0}", tempDirectory); IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast. } } catch (Exception ex) { //it is not critical if we fail to delete the temp folder Trace.Warning("Failed to delete temp folder '{0}'. Exception: {1}", tempDirectory, ex); executionContext.Warning(StringUtil.Loc("FailedDeletingTempDirectory0Message1", tempDirectory, ex.Message)); } } }
private async Task StartContainerAsync(IExecutionContext executionContext, ContainerInfo container) { Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(container, nameof(container)); ArgUtil.NotNullOrEmpty(container.ContainerImage, nameof(container.ContainerImage)); Trace.Info($"Container name: {container.ContainerName}"); Trace.Info($"Container image: {container.ContainerImage}"); Trace.Info($"Container registry: {container.ContainerRegistryEndpoint.ToString()}"); Trace.Info($"Container options: {container.ContainerCreateOptions}"); Trace.Info($"Skip container image pull: {container.SkipContainerImagePull}"); foreach (var port in container.UserPortMappings) { Trace.Info($"User provided port: {port.Value}"); } foreach (var volume in container.UserMountVolumes) { Trace.Info($"User provided volume: {volume.Value}"); } // Login to private docker registry string registryServer = string.Empty; if (container.ContainerRegistryEndpoint != Guid.Empty) { var registryEndpoint = executionContext.Endpoints.FirstOrDefault(x => x.Type == "dockerregistry" && x.Id == container.ContainerRegistryEndpoint); ArgUtil.NotNull(registryEndpoint, nameof(registryEndpoint)); string username = string.Empty; string password = string.Empty; string registryType = string.Empty; registryEndpoint.Data?.TryGetValue("registrytype", out registryType); if (string.Equals(registryType, "ACR", StringComparison.OrdinalIgnoreCase)) { string loginServer = string.Empty; registryEndpoint.Authorization?.Parameters?.TryGetValue("loginServer", out loginServer); if (loginServer != null) { loginServer = loginServer.ToLower(); } registryEndpoint.Authorization?.Parameters?.TryGetValue("serviceprincipalid", out username); registryEndpoint.Authorization?.Parameters?.TryGetValue("serviceprincipalkey", out password); registryServer = $"https://{loginServer}"; } else { registryEndpoint.Authorization?.Parameters?.TryGetValue("registry", out registryServer); registryEndpoint.Authorization?.Parameters?.TryGetValue("username", out username); registryEndpoint.Authorization?.Parameters?.TryGetValue("password", out password); } ArgUtil.NotNullOrEmpty(registryServer, nameof(registryServer)); ArgUtil.NotNullOrEmpty(username, nameof(username)); ArgUtil.NotNullOrEmpty(password, nameof(password)); int loginExitCode = await _dockerManger.DockerLogin(executionContext, registryServer, username, password); if (loginExitCode != 0) { throw new InvalidOperationException($"Docker login fail with exit code {loginExitCode}"); } } try { if (!container.SkipContainerImagePull) { if (!string.IsNullOrEmpty(registryServer) && registryServer.IndexOf("index.docker.io", StringComparison.OrdinalIgnoreCase) < 0) { var registryServerUri = new Uri(registryServer); if (!container.ContainerImage.StartsWith(registryServerUri.Authority, StringComparison.OrdinalIgnoreCase)) { container.ContainerImage = $"{registryServerUri.Authority}/{container.ContainerImage}"; } } // Pull down docker image with retry up to 3 times int retryCount = 0; int pullExitCode = 0; while (retryCount < 3) { pullExitCode = await _dockerManger.DockerPull(executionContext, container.ContainerImage); if (pullExitCode == 0) { break; } else { retryCount++; if (retryCount < 3) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10)); executionContext.Warning($"Docker pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } } } if (retryCount == 3 && pullExitCode != 0) { throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}"); } } // Mount folder into container #if OS_WINDOWS container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Work), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Work)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools)))); #else string defaultWorkingDirectory = executionContext.Variables.Get(Constants.Variables.System.DefaultWorkingDirectory); if (string.IsNullOrEmpty(defaultWorkingDirectory)) { throw new NotSupportedException(StringUtil.Loc("ContainerJobRequireSystemDefaultWorkDir")); } string workingDirectory = Path.GetDirectoryName(defaultWorkingDirectory.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)); container.MountVolumes.Add(new MountVolume(container.TranslateToHostPath(workingDirectory), workingDirectory)); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Temp), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Temp)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tasks), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tasks)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)), true)); // Ensure .taskkey file exist so we can mount it. string taskKeyFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), ".taskkey"); if (!File.Exists(taskKeyFile)) { File.WriteAllText(taskKeyFile, string.Empty); } container.MountVolumes.Add(new MountVolume(taskKeyFile, container.TranslateToContainerPath(taskKeyFile))); #endif if (container.IsJobContainer) { // See if this container brings its own Node.js container.ContainerBringNodePath = await _dockerManger.DockerInspect(context : executionContext, dockerObject : container.ContainerImage, options : $"--format=\"{{{{index .Config.Labels \\\"{_nodeJsPathLabel}\\\"}}}}\""); string node; if (!string.IsNullOrEmpty(container.ContainerBringNodePath)) { node = container.ContainerBringNodePath; } else { node = container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node", "bin", $"node{IOUtil.ExeExtension}")); } string sleepCommand = $"\"{node}\" -e \"setInterval(function(){{}}, 24 * 60 * 60 * 1000);\""; container.ContainerCommand = sleepCommand; } container.ContainerId = await _dockerManger.DockerCreate(executionContext, container); ArgUtil.NotNullOrEmpty(container.ContainerId, nameof(container.ContainerId)); if (container.IsJobContainer) { executionContext.Variables.Set(Constants.Variables.Agent.ContainerId, container.ContainerId); } // Start container int startExitCode = await _dockerManger.DockerStart(executionContext, container.ContainerId); if (startExitCode != 0) { throw new InvalidOperationException($"Docker start fail with exit code {startExitCode}"); } } finally { // Logout for private registry if (!string.IsNullOrEmpty(registryServer)) { int logoutExitCode = await _dockerManger.DockerLogout(executionContext, registryServer); if (logoutExitCode != 0) { executionContext.Error($"Docker logout fail with exit code {logoutExitCode}"); } } } try { // Make sure container is up and running var psOutputs = await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --filter status=running --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\""); if (psOutputs.FirstOrDefault(x => !string.IsNullOrEmpty(x))?.StartsWith(container.ContainerId) != true) { // container is not up and running, pull docker log for this container. await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\""); int logsExitCode = await _dockerManger.DockerLogs(executionContext, container.ContainerId); if (logsExitCode != 0) { executionContext.Warning($"Docker logs fail with exit code {logsExitCode}"); } executionContext.Warning($"Docker container {container.ContainerId} is not in running state."); } } catch (Exception ex) { // pull container log is best effort. Trace.Error("Catch exception when check container log and container status."); Trace.Error(ex); } // Get port mappings of running container if (executionContext.Container == null && !container.IsJobContainer) { container.AddPortMappings(await _dockerManger.DockerPort(executionContext, container.ContainerId)); foreach (var port in container.PortMappings) { executionContext.Variables.Set( $"{Constants.Variables.Agent.ServicePortPrefix}.{container.ContainerNetworkAlias}.ports.{port.ContainerPort}", $"{port.HostPort}"); } } #if !OS_WINDOWS if (container.IsJobContainer) { // Ensure bash exist in the image int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"sh -c \"command -v bash\""); if (execWhichBashExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execWhichBashExitCode}"); } // Get current username container.CurrentUserName = (await ExecuteCommandAsync(executionContext, "whoami", string.Empty)).FirstOrDefault(); ArgUtil.NotNullOrEmpty(container.CurrentUserName, nameof(container.CurrentUserName)); // Get current userId container.CurrentUserId = (await ExecuteCommandAsync(executionContext, "id", $"-u {container.CurrentUserName}")).FirstOrDefault(); ArgUtil.NotNullOrEmpty(container.CurrentUserId, nameof(container.CurrentUserId)); executionContext.Output(StringUtil.Loc("CreateUserWithSameUIDInsideContainer", container.CurrentUserId)); // Create an user with same uid as the agent run as user inside the container. // All command execute in docker will run as Root by default, // this will cause the agent on the host machine doesn't have permission to any new file/folder created inside the container. // So, we create a user account with same UID inside the container and let all docker exec command run as that user. string containerUserName = string.Empty; // We need to find out whether there is a user with same UID inside the container List <string> userNames = new List <string>(); int execGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"grep {container.CurrentUserId} /etc/passwd | cut -f1 -d:\"", userNames); if (execGrepExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGrepExitCode}"); } if (userNames.Count > 0) { // check all potential username that might match the UID. foreach (string username in userNames) { int execIdExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"id -u {username}"); if (execIdExitCode == 0) { containerUserName = username; break; } } } // Create a new user with same UID if (string.IsNullOrEmpty(containerUserName)) { containerUserName = $"{container.CurrentUserName}_azpcontainer"; int execUseraddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"useradd -m -u {container.CurrentUserId} {containerUserName}"); if (execUseraddExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execUseraddExitCode}"); } } executionContext.Output(StringUtil.Loc("GrantContainerUserSUDOPrivilege", containerUserName)); // Create a new group for giving sudo permission int execGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd azure_pipelines_sudo"); if (execGroupaddExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGroupaddExitCode}"); } // Add the new created user to the new created sudo group. int execUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G azure_pipelines_sudo {containerUserName}"); if (execUsermodExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execUsermodExitCode}"); } // Allow the new sudo group run any sudo command without providing password. int execEchoExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"su -c \"echo '%azure_pipelines_sudo ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers\""); if (execUsermodExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execEchoExitCode}"); } bool setupDockerGroup = executionContext.Variables.GetBoolean("VSTS_SETUP_DOCKERGROUP") ?? StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("VSTS_SETUP_DOCKERGROUP"), true); if (setupDockerGroup) { executionContext.Output(StringUtil.Loc("AllowContainerUserRunDocker", containerUserName)); // Get docker.sock group id on Host string dockerSockGroupId = (await ExecuteCommandAsync(executionContext, "stat", $"-c %g /var/run/docker.sock")).FirstOrDefault(); // We need to find out whether there is a group with same GID inside the container string existingGroupName = null; List <string> groupsOutput = new List <string>(); int execGroupGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"cat /etc/group\"", groupsOutput); if (execGroupGrepExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGroupGrepExitCode}"); } if (groupsOutput.Count > 0) { // check all potential groups that might match the GID. foreach (string groupOutput in groupsOutput) { if (!string.IsNullOrEmpty(groupOutput)) { var groupSegments = groupOutput.Split(':'); if (groupSegments.Length != 4) { Trace.Warning($"Unexpected output from /etc/group: '{groupOutput}'"); } else { // the output of /etc/group should looks like `group:x:gid:` var groupName = groupSegments[0]; var groupId = groupSegments[2]; if (string.Equals(dockerSockGroupId, groupId)) { existingGroupName = groupName; break; } } } } } if (string.IsNullOrEmpty(existingGroupName)) { // create a new group with same gid existingGroupName = "azure_pipelines_docker"; int execDockerGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd -g {dockerSockGroupId} azure_pipelines_docker"); if (execDockerGroupaddExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execDockerGroupaddExitCode}"); } } // Add the new created user to the docker socket group. int execGroupUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G {existingGroupName} {containerUserName}"); if (execGroupUsermodExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGroupUsermodExitCode}"); } } } #endif }
public async Task DownloadFromContainerAsync( RunnerActionPluginExecutionContext context, String destination, CancellationToken cancellationToken) { // Find out all container items need to be processed List <FileContainerItem> containerItems = new List <FileContainerItem>(); int retryCount = 0; while (retryCount < 3) { try { containerItems = await _fileContainerHttpClient.QueryContainerItemsAsync(_containerId, _projectId, _containerPath, cancellationToken : cancellationToken); break; } catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested) { context.Debug($"Container query has been cancelled."); throw; } catch (Exception ex) when(retryCount < 2) { retryCount++; context.Warning($"Fail to query container items under #/{_containerId}/{_containerPath}, Error: {ex.Message}"); context.Debug(ex.ToString()); } var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15)); context.Warning($"Back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } if (containerItems.Count == 0) { context.Output($"There is nothing under #/{_containerId}/{_containerPath}"); return; } // container items will include both folders, files and even file with zero size // Create all required empty folders and emptry files, gather a list of files that we need to download from server. int foldersCreated = 0; int emptryFilesCreated = 0; List <DownloadInfo> downloadFiles = new List <DownloadInfo>(); foreach (var item in containerItems.OrderBy(x => x.Path)) { if (!item.Path.StartsWith(_containerPath, StringComparison.OrdinalIgnoreCase)) { throw new ArgumentOutOfRangeException($"Item {item.Path} is not under #/{_containerId}/{_containerPath}"); } var localRelativePath = item.Path.Substring(_containerPath.Length).TrimStart('/'); var localPath = Path.Combine(destination, localRelativePath); if (item.ItemType == ContainerItemType.Folder) { context.Debug($"Ensure folder exists: {localPath}"); Directory.CreateDirectory(localPath); foldersCreated++; } else if (item.ItemType == ContainerItemType.File) { if (item.FileLength == 0) { context.Debug($"Create empty file at: {localPath}"); var parentDirectory = Path.GetDirectoryName(localPath); Directory.CreateDirectory(parentDirectory); IOUtil.DeleteFile(localPath); using (new FileStream(localPath, FileMode.Create)) { } emptryFilesCreated++; } else { context.Debug($"Prepare download {item.Path} to {localPath}"); downloadFiles.Add(new DownloadInfo(item.Path, localPath)); } } else { throw new NotSupportedException(item.ItemType.ToString()); } } if (foldersCreated > 0) { context.Output($"{foldersCreated} folders created."); } if (emptryFilesCreated > 0) { context.Output($"{emptryFilesCreated} empty files created."); } if (downloadFiles.Count == 0) { context.Output($"There is nothing to download"); return; } // Start multi-task to download all files. using (_downloadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) { // try download all files for the first time. DownloadResult downloadResult = await ParallelDownloadAsync(context, downloadFiles.AsReadOnly(), Math.Min(downloadFiles.Count, Environment.ProcessorCount), _downloadCancellationTokenSource.Token); if (downloadResult.FailedFiles.Count == 0) { // all files have been download succeed. context.Output($"{downloadFiles.Count} files download succeed."); return; } else { context.Output($"{downloadResult.FailedFiles.Count} files failed to download, retry these files after a minute."); } // Delay 1 min then retry failed files. for (int timer = 60; timer > 0; timer -= 5) { context.Output($"Retry file download after {timer} seconds."); await Task.Delay(TimeSpan.FromSeconds(5), _uploadCancellationTokenSource.Token); } // Retry download all failed files. context.Output($"Start retry {downloadResult.FailedFiles.Count} failed files upload."); DownloadResult retryDownloadResult = await ParallelDownloadAsync(context, downloadResult.FailedFiles.AsReadOnly(), Math.Min(downloadResult.FailedFiles.Count, Environment.ProcessorCount), _downloadCancellationTokenSource.Token); if (retryDownloadResult.FailedFiles.Count == 0) { // all files have been download succeed after retry. context.Output($"{downloadResult.FailedFiles} files download succeed after retry."); return; } else { throw new Exception($"{retryDownloadResult.FailedFiles.Count} files failed to download even after retry."); } } }
private async Task <DownloadResult> DownloadAsync(RunnerActionPluginExecutionContext context, int downloaderId, CancellationToken token) { List <DownloadInfo> failedFiles = new List <DownloadInfo>(); Stopwatch downloadTimer = new Stopwatch(); while (_fileDownloadQueue.TryDequeue(out DownloadInfo fileToDownload)) { token.ThrowIfCancellationRequested(); try { int retryCount = 0; bool downloadFailed = false; while (true) { try { context.Debug($"Start downloading file: '{fileToDownload.ItemPath}' (Downloader {downloaderId})"); downloadTimer.Restart(); using (FileStream fs = new FileStream(fileToDownload.LocalPath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true)) using (var downloadStream = await _fileContainerHttpClient.DownloadFileAsync(_containerId, fileToDownload.ItemPath, token, _projectId)) { await downloadStream.CopyToAsync(fs, _defaultCopyBufferSize, token); await fs.FlushAsync(token); downloadTimer.Stop(); context.Debug($"File: '{fileToDownload.LocalPath}' took {downloadTimer.ElapsedMilliseconds} milliseconds to finish download (Downloader {downloaderId})"); break; } } catch (OperationCanceledException) when(token.IsCancellationRequested) { context.Debug($"Download has been cancelled while downloading {fileToDownload.ItemPath}. (Downloader {downloaderId})"); throw; } catch (Exception ex) { retryCount++; context.Warning($"Fail to download '{fileToDownload.ItemPath}', error: {ex.Message} (Downloader {downloaderId})"); context.Debug(ex.ToString()); } if (retryCount < 3) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30)); context.Warning($"Back off {backOff.TotalSeconds} seconds before retry. (Downloader {downloaderId})"); await Task.Delay(backOff); } else { // upload still failed after 3 tries. downloadFailed = true; break; } } if (downloadFailed) { // tracking file that failed to download. failedFiles.Add(fileToDownload); } Interlocked.Increment(ref _downloadFilesProcessed); } catch (Exception ex) { // We should never context.Error($"Error '{ex.Message}' when downloading file '{fileToDownload}'. (Downloader {downloaderId})"); throw ex; } } return(new DownloadResult(failedFiles)); }