// git lfs fetch origin [ref] public async Task <int> GitLFSFetch(AgentTaskPluginExecutionContext context, string repositoryPath, string remoteName, string refSpec, string additionalCommandLine, CancellationToken cancellationToken) { context.Debug($"Fetch LFS objects for git repository at: {repositoryPath} remote: {remoteName}."); // default options for git lfs fetch. string options = StringUtil.Format($"fetch origin {refSpec}"); int retryCount = 0; int fetchExitCode = 0; while (retryCount < 3) { fetchExitCode = await ExecuteGitCommandAsync(context, repositoryPath, "lfs", options, additionalCommandLine, cancellationToken); if (fetchExitCode == 0) { break; } else { if (++retryCount < 3) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10)); context.Warning($"Git lfs fetch failed with exit code {fetchExitCode}, back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } } } return(fetchExitCode); }
private async Task ContainerHealthcheck(IExecutionContext executionContext, ContainerInfo container) { string healthCheck = "--format=\"{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}\""; string serviceHealth = await _dockerManger.DockerInspect(context : executionContext, dockerObject : container.ContainerId, options : healthCheck); if (string.IsNullOrEmpty(serviceHealth)) { // Container has no HEALTHCHECK return; } var retryCount = 0; while (string.Equals(serviceHealth, "starting", StringComparison.OrdinalIgnoreCase)) { TimeSpan backoff = BackoffTimerHelper.GetExponentialBackoff(retryCount, TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(32), TimeSpan.FromSeconds(2)); executionContext.Output($"{container.ContainerNetworkAlias} service is starting, waiting {backoff.Seconds} seconds before checking again."); await Task.Delay(backoff, executionContext.CancellationToken); serviceHealth = await _dockerManger.DockerInspect(context : executionContext, dockerObject : container.ContainerId, options : healthCheck); retryCount++; } if (string.Equals(serviceHealth, "healthy", StringComparison.OrdinalIgnoreCase)) { executionContext.Output($"{container.ContainerNetworkAlias} service is healthy."); } else { throw new InvalidOperationException($"Failed to initialize, {container.ContainerNetworkAlias} service is {serviceHealth}."); } }
// git lfs pull public async Task <int> GitLFSPull(RunnerActionPluginExecutionContext context, string repositoryPath, string additionalCommandLine, CancellationToken cancellationToken) { context.Debug($"Download LFS objects for git repository at: {repositoryPath}."); int retryCount = 0; int pullExitCode = 0; while (retryCount < 3) { pullExitCode = await ExecuteGitCommandAsync(context, repositoryPath, "lfs", "pull", additionalCommandLine, cancellationToken); if (pullExitCode == 0) { break; } else { if (++retryCount < 3) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10)); context.Warning($"Git lfs pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } } } return(pullExitCode); }
private async Task BuildActionContainerAsync(IExecutionContext executionContext, object data) { var setupInfo = data as ContainerSetupInfo; ArgUtil.NotNull(setupInfo, nameof(setupInfo)); ArgUtil.NotNullOrEmpty(setupInfo.Container.Dockerfile, nameof(setupInfo.Container.Dockerfile)); executionContext.Output($"Build container for action use: '{setupInfo.Container.Dockerfile}'."); // Build docker image with retry up to 3 times var dockerManger = HostContext.GetService <IDockerCommandManager>(); int retryCount = 0; int buildExitCode = 0; var imageName = $"{dockerManger.DockerInstanceLabel}:{Guid.NewGuid().ToString("N")}"; while (retryCount < 3) { buildExitCode = await dockerManger.DockerBuild( executionContext, setupInfo.Container.WorkingDirectory, setupInfo.Container.Dockerfile, Directory.GetParent(setupInfo.Container.Dockerfile).FullName, imageName); if (buildExitCode == 0) { break; } else { retryCount++; if (retryCount < 3) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10)); executionContext.Warning($"Docker build failed with exit code {buildExitCode}, back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } } } if (retryCount == 3 && buildExitCode != 0) { throw new InvalidOperationException($"Docker build failed with exit code {buildExitCode}"); } foreach (var stepId in setupInfo.StepIds) { CachedActionContainers[stepId] = new ContainerInfo() { ContainerImage = imageName }; Trace.Info($"Prepared docker image '{imageName}' for action {stepId} ({setupInfo.Container.Dockerfile})"); } }
// git fetch --no-tags --prune --progress --no-recurse-submodules [--depth=15] origin [+refs/pull/*:refs/remote/pull/*] [+refs/tags/1:refs/tags/1] public async Task <int> GitFetchNoTags(RunnerActionPluginExecutionContext context, string repositoryPath, string remoteName, int fetchDepth, List <string> refSpec, string additionalCommandLine, CancellationToken cancellationToken) { context.Debug($"Fetch git repository at: {repositoryPath} remote: {remoteName}."); if (refSpec != null && refSpec.Count > 0) { refSpec = refSpec.Where(r => !string.IsNullOrEmpty(r)).ToList(); } string options; // If shallow fetch add --depth arg // If the local repository is shallowed but there is no fetch depth provide for this build, // add --unshallow to convert the shallow repository to a complete repository if (fetchDepth > 0) { options = StringUtil.Format($"--no-tags --prune --progress --no-recurse-submodules --depth={fetchDepth} {remoteName} {string.Join(" ", refSpec)}"); } else if (File.Exists(Path.Combine(repositoryPath, ".git", "shallow"))) { options = StringUtil.Format($"--no-tags --prune --progress --no-recurse-submodules --unshallow {remoteName} {string.Join(" ", refSpec)}"); } else { // default options for git fetch. options = StringUtil.Format($"--no-tags --prune --progress --no-recurse-submodules {remoteName} {string.Join(" ", refSpec)}"); } int retryCount = 0; int fetchExitCode = 0; while (retryCount < 3) { fetchExitCode = await ExecuteGitCommandAsync(context, repositoryPath, "fetch", options, additionalCommandLine, cancellationToken); if (fetchExitCode == 0) { break; } else { if (++retryCount < 3) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10)); context.Warning($"Git fetch failed with exit code {fetchExitCode}, back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } } } return(fetchExitCode); }
private async Task PullActionContainerAsync(IExecutionContext executionContext, object data) { var setupInfo = data as ContainerSetupInfo; ArgUtil.NotNull(setupInfo, nameof(setupInfo)); ArgUtil.NotNullOrEmpty(setupInfo.Container.Image, nameof(setupInfo.Container.Image)); executionContext.Output($"Pull down action image '{setupInfo.Container.Image}'"); // Pull down docker image with retry up to 3 times var dockerManger = HostContext.GetService <IDockerCommandManager>(); int retryCount = 0; int pullExitCode = 0; while (retryCount < 3) { pullExitCode = await dockerManger.DockerPull(executionContext, setupInfo.Container.Image); if (pullExitCode == 0) { break; } else { retryCount++; if (retryCount < 3) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10)); executionContext.Warning($"Docker pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } } } if (retryCount == 3 && pullExitCode != 0) { throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}"); } foreach (var stepId in setupInfo.StepIds) { CachedActionContainers[stepId] = new ContainerInfo() { ContainerImage = setupInfo.Container.Image }; Trace.Info($"Prepared docker image '{setupInfo.Container.Image}' for action {stepId} ({setupInfo.Container.Image})"); } }
public async Task ConnectAsync(VssConnection jobConnection) { _connection = jobConnection; int totalAttempts = 5; int attemptCount = totalAttempts; var configurationStore = HostContext.GetService <IConfigurationStore>(); var runnerSettings = configurationStore.GetSettings(); while (!_connection.HasAuthenticated && attemptCount-- > 0) { try { await _connection.ConnectAsync(); break; } catch (Exception ex) when(attemptCount > 0) { Trace.Info($"Catch exception during connect. {attemptCount} attempts left."); Trace.Error(ex); if (runnerSettings.IsHostedServer) { await CheckNetworkEndpointsAsync(attemptCount); } } int attempt = totalAttempts - attemptCount; TimeSpan backoff = BackoffTimerHelper.GetExponentialBackoff(attempt, TimeSpan.FromMilliseconds(100), TimeSpan.FromSeconds(3.2), TimeSpan.FromMilliseconds(100)); await Task.Delay(backoff); } _taskClient = _connection.GetClient <TaskHttpClient>(); _hasConnection = true; }
public async Task <HttpResponseMessage> UploadFileAsync( Int64 containerId, String itemPath, Stream fileStream, byte[] contentId, Int64 fileLength, Boolean isGzipped, Guid scopeIdentifier, CancellationToken cancellationToken = default(CancellationToken), int chunkSize = c_defaultChunkSize, int chunkRetryTimes = c_defaultChunkRetryTimes, bool uploadFirstChunk = false, Object userState = null) { if (containerId < 1) { throw new ArgumentException(WebApiResources.ContainerIdMustBeGreaterThanZero(), "containerId"); } if (chunkSize > c_maxChunkSize) { chunkSize = c_maxChunkSize; } // if a contentId is specified but the chunk size is not a 2mb multiple error if (contentId != null && (chunkSize % c_ContentChunkMultiple) != 0) { throw new ArgumentException(FileContainerResources.ChunksizeWrongWithContentId(c_ContentChunkMultiple), "chunkSize"); } ArgumentUtility.CheckForNull(fileStream, "fileStream"); ApiResourceVersion gzipSupportedVersion = new ApiResourceVersion(new Version(1, 0), 2); ApiResourceVersion requestVersion = await NegotiateRequestVersionAsync(FileContainerResourceIds.FileContainer, s_currentApiVersion, userState, cancellationToken).ConfigureAwait(false); if (isGzipped && (requestVersion.ApiVersion < gzipSupportedVersion.ApiVersion || (requestVersion.ApiVersion == gzipSupportedVersion.ApiVersion && requestVersion.ResourceVersion < gzipSupportedVersion.ResourceVersion))) { throw new ArgumentException(FileContainerResources.GzipNotSupportedOnServer(), "isGzipped"); } if (isGzipped && fileStream.Length >= fileLength) { throw new ArgumentException(FileContainerResources.BadCompression(), "fileLength"); } HttpRequestMessage requestMessage = null; List <KeyValuePair <String, String> > query = AppendItemQueryString(itemPath, scopeIdentifier); if (fileStream.Length == 0) { // zero byte upload FileUploadTrace(itemPath, $"Upload zero byte file '{itemPath}'."); requestMessage = await CreateRequestMessageAsync(HttpMethod.Put, FileContainerResourceIds.FileContainer, routeValues : new { containerId = containerId }, version : s_currentApiVersion, queryParameters : query, userState : userState, cancellationToken : cancellationToken).ConfigureAwait(false); return(await SendAsync(requestMessage, userState, cancellationToken).ConfigureAwait(false)); } bool multiChunk = false; int totalChunks = 1; if (fileStream.Length > chunkSize) { totalChunks = (int)Math.Ceiling(fileStream.Length / (double)chunkSize); FileUploadTrace(itemPath, $"Begin chunking upload file '{itemPath}', chunk size '{chunkSize} Bytes', total chunks '{totalChunks}'."); multiChunk = true; } else { FileUploadTrace(itemPath, $"File '{itemPath}' will be uploaded in one chunk."); chunkSize = (int)fileStream.Length; } StreamParser streamParser = new StreamParser(fileStream, chunkSize); SubStream currentStream = streamParser.GetNextStream(); HttpResponseMessage response = null; Byte[] dataToSend = new Byte[chunkSize]; int currentChunk = 0; Stopwatch uploadTimer = new Stopwatch(); while (currentStream.Length > 0 && !cancellationToken.IsCancellationRequested) { currentChunk++; for (int attempt = 1; attempt <= chunkRetryTimes && !cancellationToken.IsCancellationRequested; attempt++) { if (attempt > 1) { TimeSpan backoff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(10)); FileUploadTrace(itemPath, $"Backoff {backoff.TotalSeconds} seconds before attempt '{attempt}' chunk '{currentChunk}' of file '{itemPath}'."); await Task.Delay(backoff, cancellationToken).ConfigureAwait(false); currentStream.Seek(0, SeekOrigin.Begin); } FileUploadTrace(itemPath, $"Attempt '{attempt}' for uploading chunk '{currentChunk}' of file '{itemPath}'."); // inorder for the upload to be retryable, we need the content to be re-readable // to ensure this we copy the chunk into a byte array and send that // chunk size ensures we can convert the length to an int int bytesToCopy = (int)currentStream.Length; using (MemoryStream ms = new MemoryStream(dataToSend)) { await currentStream.CopyToAsync(ms, bytesToCopy, cancellationToken).ConfigureAwait(false); } // set the content and the Content-Range header HttpContent byteArrayContent = new ByteArrayContent(dataToSend, 0, bytesToCopy); byteArrayContent.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/octet-stream"); byteArrayContent.Headers.ContentLength = currentStream.Length; byteArrayContent.Headers.ContentRange = new System.Net.Http.Headers.ContentRangeHeaderValue(currentStream.StartingPostionOnOuterStream, currentStream.EndingPostionOnOuterStream, streamParser.Length); FileUploadTrace(itemPath, $"Generate new HttpRequest for uploading file '{itemPath}', chunk '{currentChunk}' of '{totalChunks}'."); try { if (requestMessage != null) { requestMessage.Dispose(); requestMessage = null; } requestMessage = await CreateRequestMessageAsync( HttpMethod.Put, FileContainerResourceIds.FileContainer, routeValues : new { containerId = containerId }, version : s_currentApiVersion, content : byteArrayContent, queryParameters : query, userState : userState, cancellationToken : cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested) { // stop re-try on cancellation. throw; } catch (Exception ex) when(attempt < chunkRetryTimes) // not the last attempt { FileUploadTrace(itemPath, $"Chunk '{currentChunk}' attempt '{attempt}' of file '{itemPath}' fail to create HttpRequest. Error: {ex.ToString()}."); continue; } if (isGzipped) { //add gzip header info byteArrayContent.Headers.ContentEncoding.Add("gzip"); byteArrayContent.Headers.Add("x-tfs-filelength", fileLength.ToString(System.Globalization.CultureInfo.InvariantCulture)); } if (contentId != null) { byteArrayContent.Headers.Add("x-vso-contentId", Convert.ToBase64String(contentId)); // Base64FormattingOptions.None is default when not supplied } FileUploadTrace(itemPath, $"Start uploading file '{itemPath}' to server, chunk '{currentChunk}'."); uploadTimer.Restart(); try { if (response != null) { response.Dispose(); response = null; } response = await SendAsync(requestMessage, userState, cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested) { // stop re-try on cancellation. throw; } catch (Exception ex) when(attempt < chunkRetryTimes) // not the last attempt { FileUploadTrace(itemPath, $"Chunk '{currentChunk}' attempt '{attempt}' of file '{itemPath}' fail to send request to server. Error: {ex.ToString()}."); continue; } uploadTimer.Stop(); FileUploadTrace(itemPath, $"Finished upload chunk '{currentChunk}' of file '{itemPath}', elapsed {uploadTimer.ElapsedMilliseconds} (ms), response code '{response.StatusCode}'."); if (multiChunk) { FileUploadProgress(itemPath, currentChunk, (int)Math.Ceiling(fileStream.Length / (double)chunkSize)); } if (response.IsSuccessStatusCode) { break; } else if (IsFastFailResponse(response)) { FileUploadTrace(itemPath, $"Chunk '{currentChunk}' attempt '{attempt}' of file '{itemPath}' received non-success status code {response.StatusCode} for sending request and cannot continue."); break; } else { FileUploadTrace(itemPath, $"Chunk '{currentChunk}' attempt '{attempt}' of file '{itemPath}' received non-success status code {response.StatusCode} for sending request."); continue; } } // if we don't have success then bail and return the failed response if (!response.IsSuccessStatusCode) { break; } if (contentId != null && response.StatusCode == HttpStatusCode.Created) { // no need to keep uploading since the server said it has all the content FileUploadTrace(itemPath, $"Stop chunking upload the rest of the file '{itemPath}', since server already has all the content."); break; } currentStream = streamParser.GetNextStream(); if (uploadFirstChunk) { break; } } cancellationToken.ThrowIfCancellationRequested(); return(response); }
public async Task StartContainerAsync(IExecutionContext executionContext, object data) { Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); ContainerInfo container = data as ContainerInfo; ArgUtil.NotNull(container, nameof(container)); ArgUtil.NotNullOrEmpty(container.ContainerImage, nameof(container.ContainerImage)); Trace.Info($"Container name: {container.ContainerName}"); Trace.Info($"Container image: {container.ContainerImage}"); Trace.Info($"Container registry: {container.ContainerRegistryEndpoint.ToString()}"); Trace.Info($"Container options: {container.ContainerCreateOptions}"); Trace.Info($"Skip container image pull: {container.SkipContainerImagePull}"); // Check whether we are inside a container. // Our container feature requires to map working directory from host to the container. // If we are already inside a container, we will not able to find out the real working direcotry path on the host. #if OS_WINDOWS // service CExecSvc is Container Execution Agent. ServiceController[] scServices = ServiceController.GetServices(); if (scServices.Any(x => String.Equals(x.ServiceName, "cexecsvc", StringComparison.OrdinalIgnoreCase) && x.Status == ServiceControllerStatus.Running)) { throw new NotSupportedException(StringUtil.Loc("AgentAlreadyInsideContainer")); } #else var initProcessCgroup = File.ReadLines("/proc/1/cgroup"); if (initProcessCgroup.Any(x => x.IndexOf(":/docker/", StringComparison.OrdinalIgnoreCase) >= 0)) { throw new NotSupportedException(StringUtil.Loc("AgentAlreadyInsideContainer")); } #endif #if OS_WINDOWS // Check OS version (Windows server 1803 is required) object windowsInstallationType = Registry.GetValue(@"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion", "InstallationType", defaultValue: null); ArgUtil.NotNull(windowsInstallationType, nameof(windowsInstallationType)); object windowsReleaseId = Registry.GetValue(@"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion", "ReleaseId", defaultValue: null); ArgUtil.NotNull(windowsReleaseId, nameof(windowsReleaseId)); executionContext.Debug($"Current Windows version: '{windowsReleaseId} ({windowsInstallationType})'"); if (int.TryParse(windowsReleaseId.ToString(), out int releaseId)) { if (!windowsInstallationType.ToString().StartsWith("Server", StringComparison.OrdinalIgnoreCase) || releaseId < 1803) { throw new NotSupportedException(StringUtil.Loc("ContainerWindowsVersionRequirement")); } } else { throw new ArgumentOutOfRangeException(@"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\ReleaseId"); } #endif // Check docker client/server version DockerVersion dockerVersion = await _dockerManger.DockerVersion(executionContext); ArgUtil.NotNull(dockerVersion.ServerVersion, nameof(dockerVersion.ServerVersion)); ArgUtil.NotNull(dockerVersion.ClientVersion, nameof(dockerVersion.ClientVersion)); #if OS_WINDOWS Version requiredDockerVersion = new Version(17, 6); #else Version requiredDockerVersion = new Version(17, 12); #endif if (dockerVersion.ServerVersion < requiredDockerVersion) { throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerServerVersion", requiredDockerVersion, _dockerManger.DockerPath, dockerVersion.ServerVersion)); } if (dockerVersion.ClientVersion < requiredDockerVersion) { throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerClientVersion", requiredDockerVersion, _dockerManger.DockerPath, dockerVersion.ClientVersion)); } // Login to private docker registry string registryServer = string.Empty; if (container.ContainerRegistryEndpoint != Guid.Empty) { var registryEndpoint = executionContext.Endpoints.FirstOrDefault(x => x.Type == "dockerregistry" && x.Id == container.ContainerRegistryEndpoint); ArgUtil.NotNull(registryEndpoint, nameof(registryEndpoint)); string username = string.Empty; string password = string.Empty; registryEndpoint.Authorization?.Parameters?.TryGetValue("registry", out registryServer); registryEndpoint.Authorization?.Parameters?.TryGetValue("username", out username); registryEndpoint.Authorization?.Parameters?.TryGetValue("password", out password); ArgUtil.NotNullOrEmpty(registryServer, nameof(registryServer)); ArgUtil.NotNullOrEmpty(username, nameof(username)); ArgUtil.NotNullOrEmpty(password, nameof(password)); int loginExitCode = await _dockerManger.DockerLogin(executionContext, registryServer, username, password); if (loginExitCode != 0) { throw new InvalidOperationException($"Docker login fail with exit code {loginExitCode}"); } } try { if (!container.SkipContainerImagePull) { if (!string.IsNullOrEmpty(registryServer) && registryServer.IndexOf("index.docker.io", StringComparison.OrdinalIgnoreCase) < 0) { var registryServerUri = new Uri(registryServer); if (!container.ContainerImage.StartsWith(registryServerUri.Authority, StringComparison.OrdinalIgnoreCase)) { container.ContainerImage = $"{registryServerUri.Authority}/{container.ContainerImage}"; } } // Pull down docker image with retry up to 3 times int retryCount = 0; int pullExitCode = 0; while (retryCount < 3) { pullExitCode = await _dockerManger.DockerPull(executionContext, container.ContainerImage); if (pullExitCode == 0) { break; } else { retryCount++; if (retryCount < 3) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10)); executionContext.Warning($"Docker pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } } } if (retryCount == 3 && pullExitCode != 0) { throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}"); } } // Mount folder into container #if OS_WINDOWS container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Work), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Work)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools)))); #else string workingDirectory = Path.GetDirectoryName(executionContext.Variables.Get(Constants.Variables.System.DefaultWorkingDirectory).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)); container.MountVolumes.Add(new MountVolume(container.TranslateToHostPath(workingDirectory), workingDirectory)); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Temp), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Temp)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tasks), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tasks)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)), true)); // Ensure .taskkey file exist so we can mount it. string taskKeyFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), ".taskkey"); if (!File.Exists(taskKeyFile)) { File.WriteAllText(taskKeyFile, string.Empty); } container.MountVolumes.Add(new MountVolume(taskKeyFile, container.TranslateToContainerPath(taskKeyFile))); #endif #if !OS_WINDOWS if (string.IsNullOrEmpty(container.ContainerNetwork)) // create network when Windows support it. { // Create local docker network for this job to avoid port conflict when multiple agents run on same machine. container.ContainerNetwork = $"vsts_network_{Guid.NewGuid().ToString("N")}"; int networkExitCode = await _dockerManger.DockerNetworkCreate(executionContext, container.ContainerNetwork); if (networkExitCode != 0) { throw new InvalidOperationException($"Docker network create fail with exit code {networkExitCode}"); } // Expose docker network to env executionContext.Variables.Set(Constants.Variables.Agent.ContainerNetwork, container.ContainerNetwork); } #endif container.ContainerId = await _dockerManger.DockerCreate(context : executionContext, displayName : container.ContainerDisplayName, image : container.ContainerImage, mountVolumes : container.MountVolumes, network : container.ContainerNetwork, options : container.ContainerCreateOptions, environment : container.ContainerEnvironmentVariables); ArgUtil.NotNullOrEmpty(container.ContainerId, nameof(container.ContainerId)); executionContext.Variables.Set(Constants.Variables.Agent.ContainerId, container.ContainerId); // Start container int startExitCode = await _dockerManger.DockerStart(executionContext, container.ContainerId); if (startExitCode != 0) { throw new InvalidOperationException($"Docker start fail with exit code {startExitCode}"); } } finally { // Logout for private registry if (!string.IsNullOrEmpty(registryServer)) { int logoutExitCode = await _dockerManger.DockerLogout(executionContext, registryServer); if (logoutExitCode != 0) { executionContext.Error($"Docker logout fail with exit code {logoutExitCode}"); } } } #if !OS_WINDOWS // Ensure bash exist in the image int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"which bash"); if (execWhichBashExitCode != 0) { try { // Make sure container is up and running var psOutputs = await _dockerManger.DockerPS(executionContext, container.ContainerId, "--filter status=running"); if (psOutputs.FirstOrDefault(x => !string.IsNullOrEmpty(x))?.StartsWith(container.ContainerId) != true) { // container is not up and running, pull docker log for this container. await _dockerManger.DockerPS(executionContext, container.ContainerId, string.Empty); int logsExitCode = await _dockerManger.DockerLogs(executionContext, container.ContainerId); if (logsExitCode != 0) { executionContext.Warning($"Docker logs fail with exit code {logsExitCode}"); } executionContext.Warning($"Docker container {container.ContainerId} is not in running state."); } } catch (Exception ex) { // pull container log is best effort. Trace.Error("Catch exception when check container log and container status."); Trace.Error(ex); } throw new InvalidOperationException($"Docker exec fail with exit code {execWhichBashExitCode}"); } // Get current username container.CurrentUserName = (await ExecuteCommandAsync(executionContext, "whoami", string.Empty)).FirstOrDefault(); ArgUtil.NotNullOrEmpty(container.CurrentUserName, nameof(container.CurrentUserName)); // Get current userId container.CurrentUserId = (await ExecuteCommandAsync(executionContext, "id", $"-u {container.CurrentUserName}")).FirstOrDefault(); ArgUtil.NotNullOrEmpty(container.CurrentUserId, nameof(container.CurrentUserId)); executionContext.Output(StringUtil.Loc("CreateUserWithSameUIDInsideContainer", container.CurrentUserId)); // Create an user with same uid as the agent run as user inside the container. // All command execute in docker will run as Root by default, // this will cause the agent on the host machine doesn't have permission to any new file/folder created inside the container. // So, we create a user account with same UID inside the container and let all docker exec command run as that user. string containerUserName = string.Empty; // We need to find out whether there is a user with same UID inside the container List <string> userNames = new List <string>(); int execGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"grep {container.CurrentUserId} /etc/passwd | cut -f1 -d:\"", userNames); if (execGrepExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGrepExitCode}"); } if (userNames.Count > 0) { // check all potential username that might match the UID. foreach (string username in userNames) { int execIdExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"id -u {username}"); if (execIdExitCode == 0) { containerUserName = username; break; } } } // Create a new user with same UID if (string.IsNullOrEmpty(containerUserName)) { containerUserName = $"{container.CurrentUserName}_VSTSContainer"; int execUseraddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"useradd -m -u {container.CurrentUserId} {containerUserName}"); if (execUseraddExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execUseraddExitCode}"); } } executionContext.Output(StringUtil.Loc("GrantContainerUserSUDOPrivilege", containerUserName)); // Create a new vsts_sudo group for giving sudo permission int execGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd VSTS_Container_SUDO"); if (execGroupaddExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGroupaddExitCode}"); } // Add the new created user to the new created VSTS_SUDO group. int execUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G VSTS_Container_SUDO {containerUserName}"); if (execUsermodExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execUsermodExitCode}"); } // Allow the new vsts_sudo group run any sudo command without providing password. int execEchoExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"su -c \"echo '%VSTS_Container_SUDO ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers\""); if (execUsermodExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execEchoExitCode}"); } #endif }
private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, Pipelines.ActionStep repositoryAction) { Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); var repositoryReference = repositoryAction.Reference as Pipelines.RepositoryPathReference; ArgUtil.NotNull(repositoryReference, nameof(repositoryReference)); if (string.Equals(repositoryReference.RepositoryType, Pipelines.PipelineConstants.SelfAlias, StringComparison.OrdinalIgnoreCase)) { Trace.Info($"Repository action is in 'self' repository."); return; } if (!string.Equals(repositoryReference.RepositoryType, Pipelines.RepositoryTypes.GitHub, StringComparison.OrdinalIgnoreCase)) { throw new NotSupportedException(repositoryReference.RepositoryType); } ArgUtil.NotNullOrEmpty(repositoryReference.Name, nameof(repositoryReference.Name)); ArgUtil.NotNullOrEmpty(repositoryReference.Ref, nameof(repositoryReference.Ref)); string destDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), repositoryReference.Name.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), repositoryReference.Ref); if (File.Exists(destDirectory + ".completed")) { executionContext.Debug($"Action '{repositoryReference.Name}@{repositoryReference.Ref}' already downloaded at '{destDirectory}'."); return; } else { // make sure we get a clean folder ready to use. IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken); Directory.CreateDirectory(destDirectory); executionContext.Output($"Download action repository '{repositoryReference.Name}@{repositoryReference.Ref}'"); } #if OS_WINDOWS string archiveLink = $"https://api.github.com/repos/{repositoryReference.Name}/zipball/{repositoryReference.Ref}"; #else string archiveLink = $"https://api.github.com/repos/{repositoryReference.Name}/tarball/{repositoryReference.Ref}"; #endif Trace.Info($"Download archive '{archiveLink}' to '{destDirectory}'."); //download and extract action in a temp folder and rename it on success string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), "_temp_" + Guid.NewGuid()); Directory.CreateDirectory(tempDirectory); #if OS_WINDOWS string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.zip"); #else string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.tar.gz"); #endif Trace.Info($"Save archive '{archiveLink}' into {archiveFile}."); try { int retryCount = 0; // Allow up to 20 * 60s for any action to be downloaded from github graph. int timeoutSeconds = 20 * 60; while (retryCount < 3) { using (var actionDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds))) using (var actionDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(actionDownloadTimeout.Token, executionContext.CancellationToken)) { try { //open zip stream in async mode using (FileStream fs = new FileStream(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true)) using (var httpClientHandler = HostContext.CreateHttpClientHandler()) using (var httpClient = new HttpClient(httpClientHandler)) { var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN"); if (string.IsNullOrEmpty(authToken)) { // TODO: Depreciate the PREVIEW_ACTION_TOKEN authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN"); } if (!string.IsNullOrEmpty(authToken)) { HostContext.SecretMasker.AddValue(authToken); var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"PAT:{authToken}")); httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken); } else { var accessToken = executionContext.GetGitHubContext("token"); var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}")); httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken); } httpClient.DefaultRequestHeaders.UserAgent.Add(HostContext.UserAgent); using (var result = await httpClient.GetStreamAsync(archiveLink)) { await result.CopyToAsync(fs, _defaultCopyBufferSize, actionDownloadCancellation.Token); await fs.FlushAsync(actionDownloadCancellation.Token); // download succeed, break out the retry loop. break; } } } catch (OperationCanceledException) when(executionContext.CancellationToken.IsCancellationRequested) { Trace.Info($"Action download has been cancelled."); throw; } catch (Exception ex) when(retryCount < 2) { retryCount++; Trace.Error($"Fail to download archive '{archiveLink}' -- Attempt: {retryCount}"); Trace.Error(ex); if (actionDownloadTimeout.Token.IsCancellationRequested) { // action download didn't finish within timeout executionContext.Warning($"Action '{archiveLink}' didn't finish download within {timeoutSeconds} seconds."); } else { executionContext.Warning($"Failed to download action '{archiveLink}'. Error {ex.Message}"); } } } if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("_GITHUB_ACTION_DOWNLOAD_NO_BACKOFF"))) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30)); executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } } ArgUtil.NotNullOrEmpty(archiveFile, nameof(archiveFile)); executionContext.Debug($"Download '{archiveLink}' to '{archiveFile}'"); var stagingDirectory = Path.Combine(tempDirectory, "_staging"); Directory.CreateDirectory(stagingDirectory); #if OS_WINDOWS ZipFile.ExtractToDirectory(archiveFile, stagingDirectory); #else string tar = WhichUtil.Which("tar", require: true, trace: Trace); // tar -xzf using (var processInvoker = HostContext.CreateService <IProcessInvoker>()) { processInvoker.OutputDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) => { if (!string.IsNullOrEmpty(args.Data)) { Trace.Info(args.Data); } }); processInvoker.ErrorDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) => { if (!string.IsNullOrEmpty(args.Data)) { Trace.Error(args.Data); } }); int exitCode = await processInvoker.ExecuteAsync(stagingDirectory, tar, $"-xzf \"{archiveFile}\"", null, executionContext.CancellationToken); if (exitCode != 0) { throw new NotSupportedException($"Can't use 'tar -xzf' extract archive file: {archiveFile}. return code: {exitCode}."); } } #endif // repository archive from github always contains a nested folder var subDirectories = new DirectoryInfo(stagingDirectory).GetDirectories(); if (subDirectories.Length != 1) { throw new InvalidOperationException($"'{archiveFile}' contains '{subDirectories.Length}' directories"); } else { executionContext.Debug($"Unwrap '{subDirectories[0].Name}' to '{destDirectory}'"); IOUtil.CopyDirectory(subDirectories[0].FullName, destDirectory, executionContext.CancellationToken); } Trace.Verbose("Create watermark file indicate action download succeed."); File.WriteAllText(destDirectory + ".completed", DateTime.UtcNow.ToString()); executionContext.Debug($"Archive '{archiveFile}' has been unzipped into '{destDirectory}'."); Trace.Info("Finished getting action repository."); } finally { try { //if the temp folder wasn't moved -> wipe it if (Directory.Exists(tempDirectory)) { Trace.Verbose("Deleting action temp folder: {0}", tempDirectory); IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast. } } catch (Exception ex) { //it is not critical if we fail to delete the temp folder Trace.Warning("Failed to delete temp folder '{0}'. Exception: {1}", tempDirectory, ex); } } }
public async Task DownloadFromContainerAsync( RunnerActionPluginExecutionContext context, String destination, CancellationToken cancellationToken) { // Find out all container items need to be processed List <FileContainerItem> containerItems = new List <FileContainerItem>(); int retryCount = 0; while (retryCount < 3) { try { containerItems = await _fileContainerHttpClient.QueryContainerItemsAsync(_containerId, _projectId, _containerPath, cancellationToken : cancellationToken); break; } catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested) { context.Debug($"Container query has been cancelled."); throw; } catch (Exception ex) when(retryCount < 2) { retryCount++; context.Warning($"Fail to query container items under #/{_containerId}/{_containerPath}, Error: {ex.Message}"); context.Debug(ex.ToString()); } var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15)); context.Warning($"Back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } if (containerItems.Count == 0) { context.Output($"There is nothing under #/{_containerId}/{_containerPath}"); return; } // container items will include both folders, files and even file with zero size // Create all required empty folders and emptry files, gather a list of files that we need to download from server. int foldersCreated = 0; int emptryFilesCreated = 0; List <DownloadInfo> downloadFiles = new List <DownloadInfo>(); foreach (var item in containerItems.OrderBy(x => x.Path)) { if (!item.Path.StartsWith(_containerPath, StringComparison.OrdinalIgnoreCase)) { throw new ArgumentOutOfRangeException($"Item {item.Path} is not under #/{_containerId}/{_containerPath}"); } var localRelativePath = item.Path.Substring(_containerPath.Length).TrimStart('/'); var localPath = Path.Combine(destination, localRelativePath); if (item.ItemType == ContainerItemType.Folder) { context.Debug($"Ensure folder exists: {localPath}"); Directory.CreateDirectory(localPath); foldersCreated++; } else if (item.ItemType == ContainerItemType.File) { if (item.FileLength == 0) { context.Debug($"Create empty file at: {localPath}"); var parentDirectory = Path.GetDirectoryName(localPath); Directory.CreateDirectory(parentDirectory); IOUtil.DeleteFile(localPath); using (new FileStream(localPath, FileMode.Create)) { } emptryFilesCreated++; } else { context.Debug($"Prepare download {item.Path} to {localPath}"); downloadFiles.Add(new DownloadInfo(item.Path, localPath)); } } else { throw new NotSupportedException(item.ItemType.ToString()); } } if (foldersCreated > 0) { context.Output($"{foldersCreated} folders created."); } if (emptryFilesCreated > 0) { context.Output($"{emptryFilesCreated} empty files created."); } if (downloadFiles.Count == 0) { context.Output($"There is nothing to download"); return; } // Start multi-task to download all files. using (_downloadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) { // try download all files for the first time. DownloadResult downloadResult = await ParallelDownloadAsync(context, downloadFiles.AsReadOnly(), Math.Min(downloadFiles.Count, Environment.ProcessorCount), _downloadCancellationTokenSource.Token); if (downloadResult.FailedFiles.Count == 0) { // all files have been download succeed. context.Output($"{downloadFiles.Count} files download succeed."); return; } else { context.Output($"{downloadResult.FailedFiles.Count} files failed to download, retry these files after a minute."); } // Delay 1 min then retry failed files. for (int timer = 60; timer > 0; timer -= 5) { context.Output($"Retry file download after {timer} seconds."); await Task.Delay(TimeSpan.FromSeconds(5), _uploadCancellationTokenSource.Token); } // Retry download all failed files. context.Output($"Start retry {downloadResult.FailedFiles.Count} failed files upload."); DownloadResult retryDownloadResult = await ParallelDownloadAsync(context, downloadResult.FailedFiles.AsReadOnly(), Math.Min(downloadResult.FailedFiles.Count, Environment.ProcessorCount), _downloadCancellationTokenSource.Token); if (retryDownloadResult.FailedFiles.Count == 0) { // all files have been download succeed after retry. context.Output($"{downloadResult.FailedFiles} files download succeed after retry."); return; } else { throw new Exception($"{retryDownloadResult.FailedFiles.Count} files failed to download even after retry."); } } }
private async Task DownloadAsync(IExecutionContext executionContext, Pipelines.TaskStepDefinitionReference task) { Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(task, nameof(task)); ArgUtil.NotNullOrEmpty(task.Version, nameof(task.Version)); var taskServer = HostContext.GetService <ITaskServer>(); // first check to see if we already have the task string destDirectory = GetDirectory(task); Trace.Info($"Ensuring task exists: ID '{task.Id}', version '{task.Version}', name '{task.Name}', directory '{destDirectory}'."); var configurationStore = HostContext.GetService <IConfigurationStore>(); AgentSettings settings = configurationStore.GetSettings(); Boolean signingEnabled = !String.IsNullOrEmpty(settings.Fingerprint); if (File.Exists(destDirectory + ".completed") && !signingEnabled) { executionContext.Debug($"Task '{task.Name}' already downloaded at '{destDirectory}'."); return; } String taskZipPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.TaskZips), $"{task.Name}_{task.Id}_{task.Version}.zip"); if (signingEnabled && File.Exists(taskZipPath)) { executionContext.Debug($"Task '{task.Name}' already downloaded at '{taskZipPath}'."); // We need to extract the zip now because the task.json metadata for the task is used in JobExtension.InitializeJob. // This is fine because we overwrite the contents at task run time. if (!File.Exists(destDirectory + ".completed")) { // The zip exists but it hasn't been extracted yet. IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken); ExtractZip(taskZipPath, destDirectory); } return; } // delete existing task folder. Trace.Verbose("Deleting task destination folder: {0}", destDirectory); IOUtil.DeleteDirectory(destDirectory, CancellationToken.None); // Inform the user that a download is taking place. The download could take a while if // the task zip is large. It would be nice to print the localized name, but it is not // available from the reference included in the job message. executionContext.Output(StringUtil.Loc("DownloadingTask0", task.Name, task.Version)); string zipFile = string.Empty; var version = new TaskVersion(task.Version); //download and extract task in a temp folder and rename it on success string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Tasks), "_temp_" + Guid.NewGuid()); try { Directory.CreateDirectory(tempDirectory); int retryCount = 0; // Allow up to 20 * 60s for any task to be downloaded from service. // Base on Kusto, the longest we have on the service today is over 850 seconds. // Timeout limit can be overwrite by environment variable if (!int.TryParse(Environment.GetEnvironmentVariable("VSTS_TASK_DOWNLOAD_TIMEOUT") ?? string.Empty, out int timeoutSeconds)) { timeoutSeconds = 20 * 60; } while (retryCount < 3) { using (var taskDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds))) using (var taskDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(taskDownloadTimeout.Token, executionContext.CancellationToken)) { try { zipFile = Path.Combine(tempDirectory, string.Format("{0}.zip", Guid.NewGuid())); //open zip stream in async mode using (FileStream fs = new FileStream(zipFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true)) using (Stream result = await taskServer.GetTaskContentZipAsync(task.Id, version, taskDownloadCancellation.Token)) { await result.CopyToAsync(fs, _defaultCopyBufferSize, taskDownloadCancellation.Token); await fs.FlushAsync(taskDownloadCancellation.Token); // download succeed, break out the retry loop. break; } } catch (OperationCanceledException) when(executionContext.CancellationToken.IsCancellationRequested) { Trace.Info($"Task download has been cancelled."); throw; } catch (Exception ex) when(retryCount < 2) { retryCount++; Trace.Error($"Fail to download task '{task.Id} ({task.Name}/{task.Version})' -- Attempt: {retryCount}"); Trace.Error(ex); if (taskDownloadTimeout.Token.IsCancellationRequested) { // task download didn't finish within timeout executionContext.Warning(StringUtil.Loc("TaskDownloadTimeout", task.Name, timeoutSeconds)); } else { executionContext.Warning(StringUtil.Loc("TaskDownloadFailed", task.Name, ex.Message)); if (ex.InnerException != null) { executionContext.Warning("Inner Exception: {ex.InnerException.Message}"); } } } } if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("VSTS_TASK_DOWNLOAD_NO_BACKOFF"))) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30)); executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } } if (signingEnabled) { Directory.CreateDirectory(HostContext.GetDirectory(WellKnownDirectory.TaskZips)); // Copy downloaded zip to the cache on disk for future extraction. executionContext.Debug($"Copying from {zipFile} to {taskZipPath}"); File.Copy(zipFile, taskZipPath); } // We need to extract the zip regardless of whether or not signing is enabled because the task.json metadata for the task is used in JobExtension.InitializeJob. // This is fine because we overwrite the contents at task run time. Directory.CreateDirectory(destDirectory); ExtractZip(zipFile, destDirectory); executionContext.Debug($"Task '{task.Name}' has been downloaded into '{(signingEnabled ? taskZipPath : destDirectory)}'."); Trace.Info("Finished getting task."); } finally { try { //if the temp folder wasn't moved -> wipe it if (Directory.Exists(tempDirectory)) { Trace.Verbose("Deleting task temp folder: {0}", tempDirectory); IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast. } } catch (Exception ex) { //it is not critical if we fail to delete the temp folder Trace.Warning("Failed to delete temp folder '{0}'. Exception: {1}", tempDirectory, ex); executionContext.Warning(StringUtil.Loc("FailedDeletingTempDirectory0Message1", tempDirectory, ex.Message)); } } }
public async Task <TaskAgentMessage> GetNextMessageAsync(CancellationToken token) { Trace.Entering(); ArgUtil.NotNull(_session, nameof(_session)); ArgUtil.NotNull(_settings, nameof(_settings)); bool encounteringError = false; int continuousError = 0; string errorMessage = string.Empty; Stopwatch heartbeat = new Stopwatch(); heartbeat.Restart(); while (true) { token.ThrowIfCancellationRequested(); TaskAgentMessage message = null; try { message = await _runnerServer.GetAgentMessageAsync(_settings.PoolId, _session.SessionId, _lastMessageId, token); // Decrypt the message body if the session is using encryption message = DecryptMessage(message); if (message != null) { _lastMessageId = message.MessageId; } if (encounteringError) //print the message once only if there was an error { _term.WriteLine($"{DateTime.UtcNow:u}: Runner reconnected."); encounteringError = false; continuousError = 0; } if (_needToCheckAuthorizationUrlUpdate && _authorizationUrlMigrationBackgroundTask?.IsCompleted == true) { if (HostContext.GetService <IJobDispatcher>().Busy || HostContext.GetService <ISelfUpdater>().Busy) { Trace.Info("Job or runner updates in progress, update credentials next time."); } else { try { var newCred = await _authorizationUrlMigrationBackgroundTask; await _runnerServer.ConnectAsync(new Uri(_settings.ServerUrl), newCred); Trace.Info("Updated connection to use migrated credential for next GetMessage call."); _useMigratedCredentials = true; _authorizationUrlMigrationBackgroundTask = null; _needToCheckAuthorizationUrlUpdate = false; } catch (Exception ex) { Trace.Error("Fail to refresh connection with new authorization url."); Trace.Error(ex); } } } if (_authorizationUrlRollbackReattemptDelayBackgroundTask?.IsCompleted == true) { try { // we rolled back to use original creds about 2 days before, now it's a good time to try migrated creds again. Trace.Info("Re-attempt to use migrated credential"); var migratedCreds = _credMgr.LoadCredentials(); await _runnerServer.ConnectAsync(new Uri(_settings.ServerUrl), migratedCreds); _useMigratedCredentials = true; _authorizationUrlRollbackReattemptDelayBackgroundTask = null; } catch (Exception ex) { Trace.Error("Fail to refresh connection with new authorization url on rollback reattempt."); Trace.Error(ex); } } } catch (OperationCanceledException) when(token.IsCancellationRequested) { Trace.Info("Get next message has been cancelled."); throw; } catch (TaskAgentAccessTokenExpiredException) { Trace.Info("Runner OAuth token has been revoked. Unable to pull message."); throw; } catch (Exception ex) { Trace.Error("Catch exception during get next message."); Trace.Error(ex); // don't retry if SkipSessionRecover = true, DT service will delete agent session to stop agent from taking more jobs. if (ex is TaskAgentSessionExpiredException && !_settings.SkipSessionRecover && await CreateSessionAsync(token)) { Trace.Info($"{nameof(TaskAgentSessionExpiredException)} received, recovered by recreate session."); } else if (!IsGetNextMessageExceptionRetriable(ex)) { if (_useMigratedCredentials) { // migrated credentials might cause lose permission during permission check, // we will force to use original credential and try again _useMigratedCredentials = false; var reattemptBackoff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromHours(24), TimeSpan.FromHours(36)); _authorizationUrlRollbackReattemptDelayBackgroundTask = HostContext.Delay(reattemptBackoff, token); // retry migrated creds in 24-36 hours. var originalCreds = _credMgr.LoadCredentials(false); await _runnerServer.ConnectAsync(new Uri(_settings.ServerUrl), originalCreds); Trace.Error("Fallback to original credentials and try again."); } else { throw; } } else { continuousError++; //retry after a random backoff to avoid service throttling //in case of there is a service error happened and all agents get kicked off of the long poll and all agent try to reconnect back at the same time. if (continuousError <= 5) { // random backoff [15, 30] _getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30), _getNextMessageRetryInterval); } else { // more aggressive backoff [30, 60] _getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(60), _getNextMessageRetryInterval); } if (!encounteringError) { //print error only on the first consecutive error _term.WriteError($"{DateTime.UtcNow:u}: Runner connect error: {ex.Message}. Retrying until reconnected."); encounteringError = true; } // re-create VssConnection before next retry await _runnerServer.RefreshConnectionAsync(RunnerConnectionType.MessageQueue, TimeSpan.FromSeconds(60)); Trace.Info("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds); await HostContext.Delay(_getNextMessageRetryInterval, token); } } if (message == null) { if (heartbeat.Elapsed > TimeSpan.FromMinutes(30)) { Trace.Info($"No message retrieved from session '{_session.SessionId}' within last 30 minutes."); heartbeat.Restart(); } else { Trace.Verbose($"No message retrieved from session '{_session.SessionId}'."); } continue; } Trace.Info($"Message '{message.MessageId}' received from session '{_session.SessionId}'."); return(message); } }
private async Task StartContainerAsync(IExecutionContext executionContext, ContainerInfo container) { Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(container, nameof(container)); ArgUtil.NotNullOrEmpty(container.ContainerImage, nameof(container.ContainerImage)); Trace.Info($"Container name: {container.ContainerName}"); Trace.Info($"Container image: {container.ContainerImage}"); Trace.Info($"Container options: {container.ContainerCreateOptions}"); var groupName = container.IsJobContainer ? "Starting job container" : $"Starting {container.ContainerNetworkAlias} service container"; executionContext.Output($"##[group]{groupName}"); foreach (var port in container.UserPortMappings) { Trace.Info($"User provided port: {port.Value}"); } foreach (var volume in container.UserMountVolumes) { Trace.Info($"User provided volume: {volume.Value}"); var mount = new MountVolume(volume.Value); if (string.Equals(mount.SourceVolumePath, "/", StringComparison.OrdinalIgnoreCase)) { executionContext.Warning($"Volume mount {volume.Value} is going to mount '/' into the container which may cause file ownership change in the entire file system and cause Actions Runner to lose permission to access the disk."); } } // Pull down docker image with retry up to 3 times int retryCount = 0; int pullExitCode = 0; while (retryCount < 3) { pullExitCode = await _dockerManger.DockerPull(executionContext, container.ContainerImage); if (pullExitCode == 0) { break; } else { retryCount++; if (retryCount < 3) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10)); executionContext.Warning($"Docker pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } } } if (retryCount == 3 && pullExitCode != 0) { throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}"); } if (container.IsJobContainer) { // Configure job container - Mount workspace and tools, set up environment, and start long running process var githubContext = executionContext.ExpressionValues["github"] as GitHubContext; ArgUtil.NotNull(githubContext, nameof(githubContext)); var workingDirectory = githubContext["workspace"] as StringContextData; ArgUtil.NotNullOrEmpty(workingDirectory, nameof(workingDirectory)); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Work), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Work)))); #if OS_WINDOWS container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)))); #else if (Environment.GetEnvironmentVariable("K8S_POD_NAME") != null) { container.MountVolumes.Add(new MountVolume(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), "__externals_copy"), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)), true)); } else { container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)), true)); } #endif container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Temp), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Temp)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Actions), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Actions)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools)))); var tempHomeDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Temp), "_github_home"); Directory.CreateDirectory(tempHomeDirectory); container.MountVolumes.Add(new MountVolume(tempHomeDirectory, "/github/home")); container.AddPathTranslateMapping(tempHomeDirectory, "/github/home"); container.ContainerEnvironmentVariables["HOME"] = container.TranslateToContainerPath(tempHomeDirectory); var tempWorkflowDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Temp), "_github_workflow"); Directory.CreateDirectory(tempWorkflowDirectory); container.MountVolumes.Add(new MountVolume(tempWorkflowDirectory, "/github/workflow")); container.AddPathTranslateMapping(tempWorkflowDirectory, "/github/workflow"); container.ContainerWorkDirectory = container.TranslateToContainerPath(workingDirectory); container.ContainerEntryPoint = "tail"; container.ContainerEntryPointArgs = "\"-f\" \"/dev/null\""; } container.ContainerId = await _dockerManger.DockerCreate(executionContext, container); ArgUtil.NotNullOrEmpty(container.ContainerId, nameof(container.ContainerId)); // Start container int startExitCode = await _dockerManger.DockerStart(executionContext, container.ContainerId); if (startExitCode != 0) { throw new InvalidOperationException($"Docker start fail with exit code {startExitCode}"); } try { // Make sure container is up and running var psOutputs = await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --filter status=running --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\""); if (psOutputs.FirstOrDefault(x => !string.IsNullOrEmpty(x))?.StartsWith(container.ContainerId) != true) { // container is not up and running, pull docker log for this container. await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\""); int logsExitCode = await _dockerManger.DockerLogs(executionContext, container.ContainerId); if (logsExitCode != 0) { executionContext.Warning($"Docker logs fail with exit code {logsExitCode}"); } executionContext.Warning($"Docker container {container.ContainerId} is not in running state."); } } catch (Exception ex) { // pull container log is best effort. Trace.Error("Catch exception when check container log and container status."); Trace.Error(ex); } // Gather runtime container information if (!container.IsJobContainer) { var service = new DictionaryContextData() { ["id"] = new StringContextData(container.ContainerId), ["ports"] = new DictionaryContextData(), ["network"] = new StringContextData(container.ContainerNetwork) }; container.AddPortMappings(await _dockerManger.DockerPort(executionContext, container.ContainerId)); foreach (var port in container.PortMappings) { (service["ports"] as DictionaryContextData)[port.ContainerPort] = new StringContextData(port.HostPort); } executionContext.JobContext.Services[container.ContainerNetworkAlias] = service; } else { var configEnvFormat = "--format \"{{range .Config.Env}}{{println .}}{{end}}\""; var containerEnv = await _dockerManger.DockerInspect(executionContext, container.ContainerId, configEnvFormat); container.ContainerRuntimePath = DockerUtil.ParsePathFromConfigEnv(containerEnv); executionContext.JobContext.Container["id"] = new StringContextData(container.ContainerId); } executionContext.Output("##[endgroup]"); }
private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, ActionDownloadDetails actionDownloadDetails, string destDirectory) { //download and extract action in a temp folder and rename it on success string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), "_temp_" + Guid.NewGuid()); Directory.CreateDirectory(tempDirectory); #if OS_WINDOWS string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.zip"); #else string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.tar.gz"); #endif string link = actionDownloadDetails.ArchiveLink; Trace.Info($"Save archive '{link}' into {archiveFile}."); try { int retryCount = 0; // Allow up to 20 * 60s for any action to be downloaded from github graph. int timeoutSeconds = 20 * 60; while (retryCount < 3) { using (var actionDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds))) using (var actionDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(actionDownloadTimeout.Token, executionContext.CancellationToken)) { try { //open zip stream in async mode using (FileStream fs = new FileStream(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true)) using (var httpClientHandler = HostContext.CreateHttpClientHandler()) using (var httpClient = new HttpClient(httpClientHandler)) { actionDownloadDetails.ConfigureAuthorization(executionContext, httpClient); httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents); using (var response = await httpClient.GetAsync(link)) { if (response.IsSuccessStatusCode) { using (var result = await response.Content.ReadAsStreamAsync()) { await result.CopyToAsync(fs, _defaultCopyBufferSize, actionDownloadCancellation.Token); await fs.FlushAsync(actionDownloadCancellation.Token); // download succeed, break out the retry loop. break; } } else if (response.StatusCode == HttpStatusCode.NotFound) { // It doesn't make sense to retry in this case, so just stop throw new ActionNotFoundException(new Uri(link)); } else { // Something else bad happened, let's go to our retry logic response.EnsureSuccessStatusCode(); } } } } catch (OperationCanceledException) when(executionContext.CancellationToken.IsCancellationRequested) { Trace.Info("Action download has been cancelled."); throw; } catch (ActionNotFoundException) { Trace.Info($"The action at '{link}' does not exist"); throw; } catch (Exception ex) when(retryCount < 2) { retryCount++; Trace.Error($"Fail to download archive '{link}' -- Attempt: {retryCount}"); Trace.Error(ex); if (actionDownloadTimeout.Token.IsCancellationRequested) { // action download didn't finish within timeout executionContext.Warning($"Action '{link}' didn't finish download within {timeoutSeconds} seconds."); } else { executionContext.Warning($"Failed to download action '{link}'. Error: {ex.Message}"); } } } if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("_GITHUB_ACTION_DOWNLOAD_NO_BACKOFF"))) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30)); executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } } ArgUtil.NotNullOrEmpty(archiveFile, nameof(archiveFile)); executionContext.Debug($"Download '{link}' to '{archiveFile}'"); var stagingDirectory = Path.Combine(tempDirectory, "_staging"); Directory.CreateDirectory(stagingDirectory); #if OS_WINDOWS ZipFile.ExtractToDirectory(archiveFile, stagingDirectory); #else string tar = WhichUtil.Which("tar", require: true, trace: Trace); // tar -xzf using (var processInvoker = HostContext.CreateService <IProcessInvoker>()) { processInvoker.OutputDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) => { if (!string.IsNullOrEmpty(args.Data)) { Trace.Info(args.Data); } }); processInvoker.ErrorDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) => { if (!string.IsNullOrEmpty(args.Data)) { Trace.Error(args.Data); } }); int exitCode = await processInvoker.ExecuteAsync(stagingDirectory, tar, $"-xzf \"{archiveFile}\"", null, executionContext.CancellationToken); if (exitCode != 0) { throw new NotSupportedException($"Can't use 'tar -xzf' extract archive file: {archiveFile}. return code: {exitCode}."); } } #endif // repository archive from github always contains a nested folder var subDirectories = new DirectoryInfo(stagingDirectory).GetDirectories(); if (subDirectories.Length != 1) { throw new InvalidOperationException($"'{archiveFile}' contains '{subDirectories.Length}' directories"); } else { executionContext.Debug($"Unwrap '{subDirectories[0].Name}' to '{destDirectory}'"); IOUtil.CopyDirectory(subDirectories[0].FullName, destDirectory, executionContext.CancellationToken); } Trace.Verbose("Create watermark file indicate action download succeed."); string watermarkFile = GetWatermarkFilePath(destDirectory); File.WriteAllText(watermarkFile, DateTime.UtcNow.ToString()); executionContext.Debug($"Archive '{archiveFile}' has been unzipped into '{destDirectory}'."); Trace.Info("Finished getting action repository."); } finally { try { //if the temp folder wasn't moved -> wipe it if (Directory.Exists(tempDirectory)) { Trace.Verbose("Deleting action temp folder: {0}", tempDirectory); IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast. } } catch (Exception ex) { //it is not critical if we fail to delete the temp folder Trace.Warning("Failed to delete temp folder '{0}'. Exception: {1}", tempDirectory, ex); } } }
private async Task PullContainerAsync(IExecutionContext executionContext, ContainerInfo container) { Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(container, nameof(container)); ArgUtil.NotNullOrEmpty(container.ContainerImage, nameof(container.ContainerImage)); Trace.Info($"Container name: {container.ContainerName}"); Trace.Info($"Container image: {container.ContainerImage}"); Trace.Info($"Container registry: {container.ContainerRegistryEndpoint.ToString()}"); Trace.Info($"Container options: {container.ContainerCreateOptions}"); Trace.Info($"Skip container image pull: {container.SkipContainerImagePull}"); // Login to private docker registry string registryServer = string.Empty; if (container.ContainerRegistryEndpoint != Guid.Empty) { var registryEndpoint = executionContext.Endpoints.FirstOrDefault(x => x.Type == "dockerregistry" && x.Id == container.ContainerRegistryEndpoint); ArgUtil.NotNull(registryEndpoint, nameof(registryEndpoint)); string username = string.Empty; string password = string.Empty; string registryType = string.Empty; registryEndpoint.Data?.TryGetValue("registrytype", out registryType); if (string.Equals(registryType, "ACR", StringComparison.OrdinalIgnoreCase)) { string loginServer = string.Empty; registryEndpoint.Authorization?.Parameters?.TryGetValue("loginServer", out loginServer); if (loginServer != null) { loginServer = loginServer.ToLower(); } registryEndpoint.Authorization?.Parameters?.TryGetValue("serviceprincipalid", out username); registryEndpoint.Authorization?.Parameters?.TryGetValue("serviceprincipalkey", out password); registryServer = $"https://{loginServer}"; } else { registryEndpoint.Authorization?.Parameters?.TryGetValue("registry", out registryServer); registryEndpoint.Authorization?.Parameters?.TryGetValue("username", out username); registryEndpoint.Authorization?.Parameters?.TryGetValue("password", out password); } ArgUtil.NotNullOrEmpty(registryServer, nameof(registryServer)); ArgUtil.NotNullOrEmpty(username, nameof(username)); ArgUtil.NotNullOrEmpty(password, nameof(password)); int loginExitCode = await _dockerManger.DockerLogin(executionContext, registryServer, username, password); if (loginExitCode != 0) { throw new InvalidOperationException($"Docker login fail with exit code {loginExitCode}"); } } try { if (!container.SkipContainerImagePull) { if (!string.IsNullOrEmpty(registryServer) && registryServer.IndexOf("index.docker.io", StringComparison.OrdinalIgnoreCase) < 0) { var registryServerUri = new Uri(registryServer); if (!container.ContainerImage.StartsWith(registryServerUri.Authority, StringComparison.OrdinalIgnoreCase)) { container.ContainerImage = $"{registryServerUri.Authority}/{container.ContainerImage}"; } } // Pull down docker image with retry up to 3 times int retryCount = 0; int pullExitCode = 0; while (retryCount < 3) { pullExitCode = await _dockerManger.DockerPull(executionContext, container.ContainerImage); if (pullExitCode == 0) { break; } else { retryCount++; if (retryCount < 3) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10)); executionContext.Warning($"Docker pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } } } if (retryCount == 3 && pullExitCode != 0) { throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}"); } } if (PlatformUtil.RunningOnMacOS) { container.ImageOS = PlatformUtil.OS.Linux; } // if running on Windows, and attempting to run linux container, require container to have node else if (PlatformUtil.RunningOnWindows) { string containerOS = await _dockerManger.DockerInspect(context : executionContext, dockerObject : container.ContainerImage, options : $"--format=\"{{{{.Os}}}}\""); if (string.Equals("linux", containerOS, StringComparison.OrdinalIgnoreCase)) { container.ImageOS = PlatformUtil.OS.Linux; } } } finally { // Logout for private registry if (!string.IsNullOrEmpty(registryServer)) { int logoutExitCode = await _dockerManger.DockerLogout(executionContext, registryServer); if (logoutExitCode != 0) { executionContext.Error($"Docker logout fail with exit code {logoutExitCode}"); } } } }
private async Task StartContainerAsync(IExecutionContext executionContext, ContainerInfo container) { Trace.Entering(); ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(container, nameof(container)); ArgUtil.NotNullOrEmpty(container.ContainerImage, nameof(container.ContainerImage)); Trace.Info($"Container name: {container.ContainerName}"); Trace.Info($"Container image: {container.ContainerImage}"); Trace.Info($"Container registry: {container.ContainerRegistryEndpoint.ToString()}"); Trace.Info($"Container options: {container.ContainerCreateOptions}"); Trace.Info($"Skip container image pull: {container.SkipContainerImagePull}"); foreach (var port in container.UserPortMappings) { Trace.Info($"User provided port: {port.Value}"); } foreach (var volume in container.UserMountVolumes) { Trace.Info($"User provided volume: {volume.Value}"); } // Login to private docker registry string registryServer = string.Empty; if (container.ContainerRegistryEndpoint != Guid.Empty) { var registryEndpoint = executionContext.Endpoints.FirstOrDefault(x => x.Type == "dockerregistry" && x.Id == container.ContainerRegistryEndpoint); ArgUtil.NotNull(registryEndpoint, nameof(registryEndpoint)); string username = string.Empty; string password = string.Empty; string registryType = string.Empty; registryEndpoint.Data?.TryGetValue("registrytype", out registryType); if (string.Equals(registryType, "ACR", StringComparison.OrdinalIgnoreCase)) { string loginServer = string.Empty; registryEndpoint.Authorization?.Parameters?.TryGetValue("loginServer", out loginServer); registryEndpoint.Authorization?.Parameters?.TryGetValue("serviceprincipalid", out username); registryEndpoint.Authorization?.Parameters?.TryGetValue("serviceprincipalkey", out password); registryServer = $"https://{loginServer}"; } else { registryEndpoint.Authorization?.Parameters?.TryGetValue("registry", out registryServer); registryEndpoint.Authorization?.Parameters?.TryGetValue("username", out username); registryEndpoint.Authorization?.Parameters?.TryGetValue("password", out password); } ArgUtil.NotNullOrEmpty(registryServer, nameof(registryServer)); ArgUtil.NotNullOrEmpty(username, nameof(username)); ArgUtil.NotNullOrEmpty(password, nameof(password)); int loginExitCode = await _dockerManger.DockerLogin(executionContext, registryServer, username, password); if (loginExitCode != 0) { throw new InvalidOperationException($"Docker login fail with exit code {loginExitCode}"); } } try { if (!container.SkipContainerImagePull) { if (!string.IsNullOrEmpty(registryServer) && registryServer.IndexOf("index.docker.io", StringComparison.OrdinalIgnoreCase) < 0) { var registryServerUri = new Uri(registryServer); if (!container.ContainerImage.StartsWith(registryServerUri.Authority, StringComparison.OrdinalIgnoreCase)) { container.ContainerImage = $"{registryServerUri.Authority}/{container.ContainerImage}"; } } // Pull down docker image with retry up to 3 times int retryCount = 0; int pullExitCode = 0; while (retryCount < 3) { pullExitCode = await _dockerManger.DockerPull(executionContext, container.ContainerImage); if (pullExitCode == 0) { break; } else { retryCount++; if (retryCount < 3) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10)); executionContext.Warning($"Docker pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } } } if (retryCount == 3 && pullExitCode != 0) { throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}"); } } // Mount folder into container #if OS_WINDOWS container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Work), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Work)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools)))); #else string defaultWorkingDirectory = executionContext.Variables.Get(Constants.Variables.System.DefaultWorkingDirectory); if (string.IsNullOrEmpty(defaultWorkingDirectory)) { throw new NotSupportedException(StringUtil.Loc("ContainerJobRequireSystemDefaultWorkDir")); } string workingDirectory = Path.GetDirectoryName(defaultWorkingDirectory.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)); container.MountVolumes.Add(new MountVolume(container.TranslateToHostPath(workingDirectory), workingDirectory)); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Temp), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Temp)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tasks), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tasks)))); container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)), true)); // Ensure .taskkey file exist so we can mount it. string taskKeyFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), ".taskkey"); if (!File.Exists(taskKeyFile)) { File.WriteAllText(taskKeyFile, string.Empty); } container.MountVolumes.Add(new MountVolume(taskKeyFile, container.TranslateToContainerPath(taskKeyFile))); #endif if (container.IsJobContainer) { // See if this container brings its own Node.js container.ContainerBringNodePath = await _dockerManger.DockerInspect(context : executionContext, dockerObject : container.ContainerImage, options : $"--format=\"{{{{index .Config.Labels \\\"{_nodeJsPathLabel}\\\"}}}}\""); string node; if (!string.IsNullOrEmpty(container.ContainerBringNodePath)) { node = container.ContainerBringNodePath; } else { node = container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node", "bin", $"node{IOUtil.ExeExtension}")); } string sleepCommand = $"\"{node}\" -e \"setInterval(function(){{}}, 24 * 60 * 60 * 1000);\""; container.ContainerCommand = sleepCommand; } container.ContainerId = await _dockerManger.DockerCreate(executionContext, container); ArgUtil.NotNullOrEmpty(container.ContainerId, nameof(container.ContainerId)); if (container.IsJobContainer) { executionContext.Variables.Set(Constants.Variables.Agent.ContainerId, container.ContainerId); } // Start container int startExitCode = await _dockerManger.DockerStart(executionContext, container.ContainerId); if (startExitCode != 0) { throw new InvalidOperationException($"Docker start fail with exit code {startExitCode}"); } } finally { // Logout for private registry if (!string.IsNullOrEmpty(registryServer)) { int logoutExitCode = await _dockerManger.DockerLogout(executionContext, registryServer); if (logoutExitCode != 0) { executionContext.Error($"Docker logout fail with exit code {logoutExitCode}"); } } } try { // Make sure container is up and running var psOutputs = await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --filter status=running --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\""); if (psOutputs.FirstOrDefault(x => !string.IsNullOrEmpty(x))?.StartsWith(container.ContainerId) != true) { // container is not up and running, pull docker log for this container. await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\""); int logsExitCode = await _dockerManger.DockerLogs(executionContext, container.ContainerId); if (logsExitCode != 0) { executionContext.Warning($"Docker logs fail with exit code {logsExitCode}"); } executionContext.Warning($"Docker container {container.ContainerId} is not in running state."); } } catch (Exception ex) { // pull container log is best effort. Trace.Error("Catch exception when check container log and container status."); Trace.Error(ex); } // Get port mappings of running container if (executionContext.Container == null && !container.IsJobContainer) { container.AddPortMappings(await _dockerManger.DockerPort(executionContext, container.ContainerId)); foreach (var port in container.PortMappings) { executionContext.Variables.Set( $"{Constants.Variables.Agent.ServicePortPrefix}.{container.ContainerNetworkAlias}.ports.{port.ContainerPort}", $"{port.HostPort}"); } } #if !OS_WINDOWS if (container.IsJobContainer) { // Ensure bash exist in the image int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"sh -c \"command -v bash\""); if (execWhichBashExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execWhichBashExitCode}"); } // Get current username container.CurrentUserName = (await ExecuteCommandAsync(executionContext, "whoami", string.Empty)).FirstOrDefault(); ArgUtil.NotNullOrEmpty(container.CurrentUserName, nameof(container.CurrentUserName)); // Get current userId container.CurrentUserId = (await ExecuteCommandAsync(executionContext, "id", $"-u {container.CurrentUserName}")).FirstOrDefault(); ArgUtil.NotNullOrEmpty(container.CurrentUserId, nameof(container.CurrentUserId)); executionContext.Output(StringUtil.Loc("CreateUserWithSameUIDInsideContainer", container.CurrentUserId)); // Create an user with same uid as the agent run as user inside the container. // All command execute in docker will run as Root by default, // this will cause the agent on the host machine doesn't have permission to any new file/folder created inside the container. // So, we create a user account with same UID inside the container and let all docker exec command run as that user. string containerUserName = string.Empty; // We need to find out whether there is a user with same UID inside the container List <string> userNames = new List <string>(); int execGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"grep {container.CurrentUserId} /etc/passwd | cut -f1 -d:\"", userNames); if (execGrepExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGrepExitCode}"); } if (userNames.Count > 0) { // check all potential username that might match the UID. foreach (string username in userNames) { int execIdExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"id -u {username}"); if (execIdExitCode == 0) { containerUserName = username; break; } } } // Create a new user with same UID if (string.IsNullOrEmpty(containerUserName)) { containerUserName = $"{container.CurrentUserName}_azpcontainer"; int execUseraddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"useradd -m -u {container.CurrentUserId} {containerUserName}"); if (execUseraddExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execUseraddExitCode}"); } } executionContext.Output(StringUtil.Loc("GrantContainerUserSUDOPrivilege", containerUserName)); // Create a new group for giving sudo permission int execGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd azure_pipelines_sudo"); if (execGroupaddExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGroupaddExitCode}"); } // Add the new created user to the new created sudo group. int execUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G azure_pipelines_sudo {containerUserName}"); if (execUsermodExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execUsermodExitCode}"); } // Allow the new sudo group run any sudo command without providing password. int execEchoExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"su -c \"echo '%azure_pipelines_sudo ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers\""); if (execUsermodExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execEchoExitCode}"); } bool setupDockerGroup = executionContext.Variables.GetBoolean("VSTS_SETUP_DOCKERGROUP") ?? StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("VSTS_SETUP_DOCKERGROUP"), true); if (setupDockerGroup) { executionContext.Output(StringUtil.Loc("AllowContainerUserRunDocker", containerUserName)); // Get docker.sock group id on Host string dockerSockGroupId = (await ExecuteCommandAsync(executionContext, "stat", $"-c %g /var/run/docker.sock")).FirstOrDefault(); // We need to find out whether there is a group with same GID inside the container string existingGroupName = null; List <string> groupsOutput = new List <string>(); int execGroupGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"cat /etc/group\"", groupsOutput); if (execGroupGrepExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGroupGrepExitCode}"); } if (groupsOutput.Count > 0) { // check all potential groups that might match the GID. foreach (string groupOutput in groupsOutput) { if (!string.IsNullOrEmpty(groupOutput)) { var groupSegments = groupOutput.Split(':'); if (groupSegments.Length != 4) { Trace.Warning($"Unexpected output from /etc/group: '{groupOutput}'"); } else { // the output of /etc/group should looks like `group:x:gid:` var groupName = groupSegments[0]; var groupId = groupSegments[2]; if (string.Equals(dockerSockGroupId, groupId)) { existingGroupName = groupName; break; } } } } } if (string.IsNullOrEmpty(existingGroupName)) { // create a new group with same gid existingGroupName = "azure_pipelines_docker"; int execDockerGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd -g {dockerSockGroupId} azure_pipelines_docker"); if (execDockerGroupaddExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execDockerGroupaddExitCode}"); } } // Add the new created user to the docker socket group. int execGroupUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G {existingGroupName} {containerUserName}"); if (execGroupUsermodExitCode != 0) { throw new InvalidOperationException($"Docker exec fail with exit code {execGroupUsermodExitCode}"); } } } #endif }
private async Task <VssCredentials> GetNewOAuthAuthorizationSetting(CancellationToken token) { Trace.Info("Start checking oauth authorization url update."); while (true) { var backoff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromMinutes(30), TimeSpan.FromMinutes(45)); await HostContext.Delay(backoff, token); try { var migratedAuthorizationUrl = await _runnerServer.GetRunnerAuthUrlAsync(_settings.PoolId, _settings.AgentId); if (!string.IsNullOrEmpty(migratedAuthorizationUrl)) { var credData = _configStore.GetCredentials(); var clientId = credData.Data.GetValueOrDefault("clientId", null); var currentAuthorizationUrl = credData.Data.GetValueOrDefault("authorizationUrl", null); Trace.Info($"Current authorization url: {currentAuthorizationUrl}, new authorization url: {migratedAuthorizationUrl}"); if (string.Equals(currentAuthorizationUrl, migratedAuthorizationUrl, StringComparison.OrdinalIgnoreCase)) { // We don't need to update credentials. Trace.Info("No needs to update authorization url"); await Task.Delay(TimeSpan.FromMilliseconds(-1), token); } var keyManager = HostContext.GetService <IRSAKeyManager>(); var signingCredentials = VssSigningCredentials.Create(() => keyManager.GetKey()); var migratedClientCredential = new VssOAuthJwtBearerClientCredential(clientId, migratedAuthorizationUrl, signingCredentials); var migratedRunnerCredential = new VssOAuthCredential(new Uri(migratedAuthorizationUrl, UriKind.Absolute), VssOAuthGrant.ClientCredentials, migratedClientCredential); Trace.Info("Try connect service with Token Service OAuth endpoint."); var runnerServer = HostContext.CreateService <IRunnerServer>(); await runnerServer.ConnectAsync(new Uri(_settings.ServerUrl), migratedRunnerCredential); await runnerServer.GetAgentPoolsAsync(); Trace.Info($"Successfully connected service with new authorization url."); var migratedCredData = new CredentialData { Scheme = Constants.Configuration.OAuth, Data = { { "clientId", clientId }, { "authorizationUrl", migratedAuthorizationUrl }, { "oauthEndpointUrl", migratedAuthorizationUrl }, }, }; _configStore.SaveMigratedCredential(migratedCredData); return(migratedRunnerCredential); } else { Trace.Verbose("No authorization url updates"); } } catch (Exception ex) { Trace.Error("Fail to get/test new authorization url."); Trace.Error(ex); try { await _runnerServer.ReportRunnerAuthUrlErrorAsync(_settings.PoolId, _settings.AgentId, ex.ToString()); } catch (Exception e) { // best effort Trace.Error("Fail to report the migration error"); Trace.Error(e); } } } }
public async Task <TaskAgentMessage> GetNextMessageAsync(CancellationToken token) { Trace.Entering(); ArgUtil.NotNull(_session, nameof(_session)); ArgUtil.NotNull(_settings, nameof(_settings)); bool encounteringError = false; int continuousError = 0; string errorMessage = string.Empty; Stopwatch heartbeat = new Stopwatch(); heartbeat.Restart(); while (true) { token.ThrowIfCancellationRequested(); TaskAgentMessage message = null; try { message = await _runnerServer.GetAgentMessageAsync(_settings.PoolId, _session.SessionId, _lastMessageId, token); // Decrypt the message body if the session is using encryption message = DecryptMessage(message); if (message != null) { _lastMessageId = message.MessageId; } if (encounteringError) //print the message once only if there was an error { _term.WriteLine($"{DateTime.UtcNow:u}: Runner reconnected."); encounteringError = false; continuousError = 0; } } catch (OperationCanceledException) when(token.IsCancellationRequested) { Trace.Info("Get next message has been cancelled."); throw; } catch (TaskAgentAccessTokenExpiredException) { Trace.Info("Runner OAuth token has been revoked. Unable to pull message."); throw; } catch (Exception ex) { Trace.Error("Catch exception during get next message."); Trace.Error(ex); // don't retry if SkipSessionRecover = true, DT service will delete agent session to stop agent from taking more jobs. if (ex is TaskAgentSessionExpiredException && !_settings.SkipSessionRecover && await CreateSessionAsync(token)) { Trace.Info($"{nameof(TaskAgentSessionExpiredException)} received, recovered by recreate session."); } else if (!IsGetNextMessageExceptionRetriable(ex)) { throw; } else { continuousError++; //retry after a random backoff to avoid service throttling //in case of there is a service error happened and all agents get kicked off of the long poll and all agent try to reconnect back at the same time. if (continuousError <= 5) { // random backoff [15, 30] _getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30), _getNextMessageRetryInterval); } else { // more aggressive backoff [30, 60] _getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(60), _getNextMessageRetryInterval); } if (!encounteringError) { //print error only on the first consecutive error _term.WriteError($"{DateTime.UtcNow:u}: Runner connect error: {ex.Message}. Retrying until reconnected."); encounteringError = true; } // re-create VssConnection before next retry await _runnerServer.RefreshConnectionAsync(RunnerConnectionType.MessageQueue, TimeSpan.FromSeconds(60)); Trace.Info("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds); await HostContext.Delay(_getNextMessageRetryInterval, token); } } if (message == null) { if (heartbeat.Elapsed > TimeSpan.FromMinutes(30)) { Trace.Info($"No message retrieved from session '{_session.SessionId}' within last 30 minutes."); heartbeat.Restart(); } else { Trace.Verbose($"No message retrieved from session '{_session.SessionId}'."); } continue; } Trace.Info($"Message '{message.MessageId}' received from session '{_session.SessionId}'."); return(message); } }
public async Task <Boolean> CreateSessionAsync(CancellationToken token) { Trace.Entering(); // Settings var configManager = HostContext.GetService <IConfigurationManager>(); _settings = configManager.LoadSettings(); var serverUrl = _settings.ServerUrl; Trace.Info(_settings); // Create connection. Trace.Info("Loading Credentials"); _useMigratedCredentials = !StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_SPSAUTHURL")); VssCredentials creds = _credMgr.LoadCredentials(_useMigratedCredentials); var agent = new TaskAgentReference { Id = _settings.AgentId, Name = _settings.AgentName, Version = BuildConstants.RunnerPackage.Version, OSDescription = RuntimeInformation.OSDescription, }; string sessionName = $"{Environment.MachineName ?? "RUNNER"}"; var taskAgentSession = new TaskAgentSession(sessionName, agent); string errorMessage = string.Empty; bool encounteringError = false; var originalCreds = _configStore.GetCredentials(); var migratedCreds = _configStore.GetMigratedCredentials(); if (migratedCreds == null) { _useMigratedCredentials = false; if (originalCreds.Scheme == Constants.Configuration.OAuth) { _needToCheckAuthorizationUrlUpdate = true; } } while (true) { token.ThrowIfCancellationRequested(); Trace.Info($"Attempt to create session."); try { Trace.Info("Connecting to the Runner Server..."); await _runnerServer.ConnectAsync(new Uri(serverUrl), creds); Trace.Info("VssConnection created"); _term.WriteLine(); _term.WriteSuccessMessage("Connected to GitHub"); _term.WriteLine(); _session = await _runnerServer.CreateAgentSessionAsync( _settings.PoolId, taskAgentSession, token); Trace.Info($"Session created."); if (encounteringError) { _term.WriteLine($"{DateTime.UtcNow:u}: Runner reconnected."); _sessionCreationExceptionTracker.Clear(); encounteringError = false; } if (_needToCheckAuthorizationUrlUpdate) { // start background task try to get new authorization url _authorizationUrlMigrationBackgroundTask = GetNewOAuthAuthorizationSetting(token); } return(true); } catch (OperationCanceledException) when(token.IsCancellationRequested) { Trace.Info("Session creation has been cancelled."); throw; } catch (TaskAgentAccessTokenExpiredException) { Trace.Info("Runner OAuth token has been revoked. Session creation failed."); throw; } catch (Exception ex) { Trace.Error("Catch exception during create session."); Trace.Error(ex); if (!IsSessionCreationExceptionRetriable(ex)) { if (_useMigratedCredentials) { // migrated credentials might cause lose permission during permission check, // we will force to use original credential and try again _useMigratedCredentials = false; var reattemptBackoff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromHours(24), TimeSpan.FromHours(36)); _authorizationUrlRollbackReattemptDelayBackgroundTask = HostContext.Delay(reattemptBackoff, token); // retry migrated creds in 24-36 hours. creds = _credMgr.LoadCredentials(false); Trace.Error("Fallback to original credentials and try again."); } else { _term.WriteError($"Failed to create session. {ex.Message}"); return(false); } } if (!encounteringError) //print the message only on the first error { _term.WriteError($"{DateTime.UtcNow:u}: Runner connect error: {ex.Message}. Retrying until reconnected."); encounteringError = true; } Trace.Info("Sleeping for {0} seconds before retrying.", _sessionCreationRetryInterval.TotalSeconds); await HostContext.Delay(_sessionCreationRetryInterval, token); } } }
// git fetch --tags --prune --progress --no-recurse-submodules [--depth=15] origin [+refs/pull/*:refs/remote/pull/*] public async Task <int> GitFetch(AgentTaskPluginExecutionContext context, string repositoryPath, string remoteName, int fetchDepth, List <string> refSpec, string additionalCommandLine, CancellationToken cancellationToken) { context.Debug($"Fetch git repository at: {repositoryPath} remote: {remoteName}."); if (refSpec != null && refSpec.Count > 0) { refSpec = refSpec.Where(r => !string.IsNullOrEmpty(r)).ToList(); } // Git 2.20 changed its fetch behavior, rejecting tag updates if the --force flag is not provided // See https://git-scm.com/docs/git-fetch for more details string forceTag = string.Empty; if (gitVersion >= new Version(2, 20)) { forceTag = "--force"; } // default options for git fetch. string options = StringUtil.Format($"{forceTag} --tags --prune --progress --no-recurse-submodules {remoteName} {string.Join(" ", refSpec)}"); // If shallow fetch add --depth arg // If the local repository is shallowed but there is no fetch depth provide for this build, // add --unshallow to convert the shallow repository to a complete repository if (fetchDepth > 0) { options = StringUtil.Format($"{forceTag} --tags --prune --progress --no-recurse-submodules --depth={fetchDepth} {remoteName} {string.Join(" ", refSpec)}"); } else { if (File.Exists(Path.Combine(repositoryPath, ".git", "shallow"))) { options = StringUtil.Format($"{forceTag} --tags --prune --progress --no-recurse-submodules --unshallow {remoteName} {string.Join(" ", refSpec)}"); } } int retryCount = 0; int fetchExitCode = 0; while (retryCount < 3) { Stopwatch watch = new Stopwatch(); watch.Start(); fetchExitCode = await ExecuteGitCommandAsync(context, repositoryPath, "fetch", options, additionalCommandLine, cancellationToken); watch.Stop(); // Publish some fetch statistics context.PublishTelemetry(area: "AzurePipelinesAgent", feature: "GitFetch", properties: new Dictionary <string, string> { { "ElapsedTimeMilliseconds", $"{watch.ElapsedMilliseconds}" }, { "RefSpec", string.Join(" ", refSpec) }, { "RemoteName", remoteName }, { "FetchDepth", $"{fetchDepth}" }, { "ExitCode", $"{fetchExitCode}" }, { "Options", options } }); if (fetchExitCode == 0) { break; } else { if (++retryCount < 3) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10)); context.Warning($"Git fetch failed with exit code {fetchExitCode}, back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } } } return(fetchExitCode); }
protected override async Task <HttpResponseMessage> SendAsync( HttpRequestMessage request, CancellationToken cancellationToken) { ArgUtil.NotNull(request, nameof(request)); if (PlatformUtil.RunningOnMacOS || PlatformUtil.RunningOnLinux) { request.Version = HttpVersion.Version11; } // Potential improvements: // 1. Calculate a max time across attempts, to avoid retries (this class) on top of retries (VssHttpRetryMessageHandler) // causing more time to pass than expected in degenerative cases. // 2. Increase the per-attempt timeout on each attempt. Instead of 5 minutes on each attempt, start low and build to 10-20 minutes. HttpResponseMessage response = null; // We can safely retry on timeout if the request isn't one that changes state. Boolean canRetry = (request.Method == HttpMethod.Get || request.Method == HttpMethod.Head || request.Method == HttpMethod.Options); if (canRetry) { Int32 attempt = 1; TimeoutException exception = null; Int32 maxAttempts = _retryOptions.MaxRetries + 1; while (attempt <= maxAttempts) { // Reset the exception so we don't have a lingering variable exception = null; Stopwatch watch = Stopwatch.StartNew(); try { response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false); break; } catch (TimeoutException ex) { exception = ex; } TimeSpan backoff; if (attempt < maxAttempts) { backoff = BackoffTimerHelper.GetExponentialBackoff( attempt, _retryOptions.MinBackoff, _retryOptions.MaxBackoff, _retryOptions.BackoffCoefficient); } else { break; } string message = StringUtil.Loc("RMContainerItemRequestTimedOut", (int)watch.Elapsed.TotalSeconds, backoff.TotalSeconds, request.Method, request.RequestUri); _logger.Warning(message); attempt++; await Task.Delay(backoff, cancellationToken).ConfigureAwait(false); } if (exception != null) { throw exception; } } else { // No retries. Just pipe the request through to the other handlers. response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false); } return(response); }
public async Task RenewJobRequestAsync(int poolId, long requestId, Guid lockToken, TaskCompletionSource <int> firstJobRequestRenewed, CancellationToken token) { var agentServer = HostContext.GetService <IAgentServer>(); TaskAgentJobRequest request = null; int firstRenewRetryLimit = 5; int encounteringError = 0; // renew lock during job running. // stop renew only if cancellation token for lock renew task been signal or exception still happen after retry. while (!token.IsCancellationRequested) { try { request = await agentServer.RenewAgentRequestAsync(poolId, requestId, lockToken, token); Trace.Info($"Successfully renew job request {requestId}, job is valid till {request.LockedUntil.Value}"); if (!firstJobRequestRenewed.Task.IsCompleted) { // fire first renew succeed event. firstJobRequestRenewed.TrySetResult(0); } if (encounteringError > 0) { encounteringError = 0; agentServer.SetConnectionTimeout(AgentConnectionType.JobRequest, TimeSpan.FromSeconds(60)); HostContext.WritePerfCounter("JobRenewRecovered"); } // renew again after 60 sec delay await HostContext.Delay(TimeSpan.FromSeconds(60), token); } catch (TaskAgentJobNotFoundException) { // no need for retry. the job is not valid anymore. Trace.Info($"TaskAgentJobNotFoundException received when renew job request {requestId}, job is no longer valid, stop renew job request."); return; } catch (TaskAgentJobTokenExpiredException) { // no need for retry. the job is not valid anymore. Trace.Info($"TaskAgentJobTokenExpiredException received renew job request {requestId}, job is no longer valid, stop renew job request."); return; } catch (OperationCanceledException) when(token.IsCancellationRequested) { // OperationCanceledException may caused by http timeout or _lockRenewalTokenSource.Cance(); // Stop renew only on cancellation token fired. Trace.Info($"job renew has been canceled, stop renew job request {requestId}."); return; } catch (Exception ex) { Trace.Error($"Catch exception during renew agent jobrequest {requestId}."); Trace.Error(ex); encounteringError++; // retry TimeSpan remainingTime = TimeSpan.Zero; if (!firstJobRequestRenewed.Task.IsCompleted) { // retry 5 times every 10 sec for the first renew if (firstRenewRetryLimit-- > 0) { remainingTime = TimeSpan.FromSeconds(10); } } else { // retry till reach lockeduntil + 5 mins extra buffer. remainingTime = request.LockedUntil.Value + TimeSpan.FromMinutes(5) - DateTime.UtcNow; } if (remainingTime > TimeSpan.Zero) { TimeSpan delayTime; if (!firstJobRequestRenewed.Task.IsCompleted) { Trace.Info($"Retrying lock renewal for jobrequest {requestId}. The first job renew request has failed."); delayTime = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10)); } else { Trace.Info($"Retrying lock renewal for jobrequest {requestId}. Job is valid until {request.LockedUntil.Value}."); if (encounteringError > 5) { delayTime = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30)); } else { delayTime = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15)); } } // Re-establish connection to server in order to avoid affinity with server. // Reduce connection timeout to 30 seconds (from 60s) HostContext.WritePerfCounter("ResetJobRenewConnection"); await agentServer.RefreshConnectionAsync(AgentConnectionType.JobRequest, TimeSpan.FromSeconds(30)); try { // back-off before next retry. await HostContext.Delay(delayTime, token); } catch (OperationCanceledException) when(token.IsCancellationRequested) { Trace.Info($"job renew has been canceled, stop renew job request {requestId}."); } } else { Trace.Info($"Lock renewal has run out of retry, stop renew lock for jobrequest {requestId}."); HostContext.WritePerfCounter("JobRenewReachLimit"); return; } } } }
private async Task <DownloadResult> DownloadAsync(RunnerActionPluginExecutionContext context, int downloaderId, CancellationToken token) { List <DownloadInfo> failedFiles = new List <DownloadInfo>(); Stopwatch downloadTimer = new Stopwatch(); while (_fileDownloadQueue.TryDequeue(out DownloadInfo fileToDownload)) { token.ThrowIfCancellationRequested(); try { int retryCount = 0; bool downloadFailed = false; while (true) { try { context.Debug($"Start downloading file: '{fileToDownload.ItemPath}' (Downloader {downloaderId})"); downloadTimer.Restart(); using (FileStream fs = new FileStream(fileToDownload.LocalPath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true)) using (var downloadStream = await _fileContainerHttpClient.DownloadFileAsync(_containerId, fileToDownload.ItemPath, token, _projectId)) { await downloadStream.CopyToAsync(fs, _defaultCopyBufferSize, token); await fs.FlushAsync(token); downloadTimer.Stop(); context.Debug($"File: '{fileToDownload.LocalPath}' took {downloadTimer.ElapsedMilliseconds} milliseconds to finish download (Downloader {downloaderId})"); break; } } catch (OperationCanceledException) when(token.IsCancellationRequested) { context.Debug($"Download has been cancelled while downloading {fileToDownload.ItemPath}. (Downloader {downloaderId})"); throw; } catch (Exception ex) { retryCount++; context.Warning($"Fail to download '{fileToDownload.ItemPath}', error: {ex.Message} (Downloader {downloaderId})"); context.Debug(ex.ToString()); } if (retryCount < 3) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30)); context.Warning($"Back off {backOff.TotalSeconds} seconds before retry. (Downloader {downloaderId})"); await Task.Delay(backOff); } else { // upload still failed after 3 tries. downloadFailed = true; break; } } if (downloadFailed) { // tracking file that failed to download. failedFiles.Add(fileToDownload); } Interlocked.Increment(ref _downloadFilesProcessed); } catch (Exception ex) { // We should never context.Error($"Error '{ex.Message}' when downloading file '{fileToDownload}'. (Downloader {downloaderId})"); throw; } } return(new DownloadResult(failedFiles)); }