// git lfs fetch origin [ref]
public async Task <int> GitLFSFetch(AgentTaskPluginExecutionContext context, string repositoryPath, string remoteName, string refSpec, string additionalCommandLine, CancellationToken cancellationToken)
{
context.Debug($"Fetch LFS objects for git repository at: {repositoryPath} remote: {remoteName}.");
// default options for git lfs fetch.
string options = StringUtil.Format($"fetch origin {refSpec}");
int retryCount = 0;
int fetchExitCode = 0;
while (retryCount < 3)
{
fetchExitCode = await ExecuteGitCommandAsync(context, repositoryPath, "lfs", options, additionalCommandLine, cancellationToken);
if (fetchExitCode == 0)
{
break;
}
else
{
if (++retryCount < 3)
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
context.Warning($"Git lfs fetch failed with exit code {fetchExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
}
return(fetchExitCode);
}
private async Task ContainerHealthcheck(IExecutionContext executionContext, ContainerInfo container)
{
string healthCheck = "--format=\"{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}\"";
string serviceHealth = await _dockerManger.DockerInspect(context : executionContext, dockerObject : container.ContainerId, options : healthCheck);
if (string.IsNullOrEmpty(serviceHealth))
{
// Container has no HEALTHCHECK
return;
}
var retryCount = 0;
while (string.Equals(serviceHealth, "starting", StringComparison.OrdinalIgnoreCase))
{
TimeSpan backoff = BackoffTimerHelper.GetExponentialBackoff(retryCount, TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(32), TimeSpan.FromSeconds(2));
executionContext.Output($"{container.ContainerNetworkAlias} service is starting, waiting {backoff.Seconds} seconds before checking again.");
await Task.Delay(backoff, executionContext.CancellationToken);
serviceHealth = await _dockerManger.DockerInspect(context : executionContext, dockerObject : container.ContainerId, options : healthCheck);
retryCount++;
}
if (string.Equals(serviceHealth, "healthy", StringComparison.OrdinalIgnoreCase))
{
executionContext.Output($"{container.ContainerNetworkAlias} service is healthy.");
}
else
{
throw new InvalidOperationException($"Failed to initialize, {container.ContainerNetworkAlias} service is {serviceHealth}.");
}
}
// git lfs pull
public async Task <int> GitLFSPull(RunnerActionPluginExecutionContext context, string repositoryPath, string additionalCommandLine, CancellationToken cancellationToken)
{
context.Debug($"Download LFS objects for git repository at: {repositoryPath}.");
int retryCount = 0;
int pullExitCode = 0;
while (retryCount < 3)
{
pullExitCode = await ExecuteGitCommandAsync(context, repositoryPath, "lfs", "pull", additionalCommandLine, cancellationToken);
if (pullExitCode == 0)
{
break;
}
else
{
if (++retryCount < 3)
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
context.Warning($"Git lfs pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
}
return(pullExitCode);
}
private async Task BuildActionContainerAsync(IExecutionContext executionContext, object data)
{
var setupInfo = data as ContainerSetupInfo;
ArgUtil.NotNull(setupInfo, nameof(setupInfo));
ArgUtil.NotNullOrEmpty(setupInfo.Container.Dockerfile, nameof(setupInfo.Container.Dockerfile));
executionContext.Output($"Build container for action use: '{setupInfo.Container.Dockerfile}'.");
// Build docker image with retry up to 3 times
var dockerManger = HostContext.GetService <IDockerCommandManager>();
int retryCount = 0;
int buildExitCode = 0;
var imageName = $"{dockerManger.DockerInstanceLabel}:{Guid.NewGuid().ToString("N")}";
while (retryCount < 3)
{
buildExitCode = await dockerManger.DockerBuild(
executionContext,
setupInfo.Container.WorkingDirectory,
setupInfo.Container.Dockerfile,
Directory.GetParent(setupInfo.Container.Dockerfile).FullName,
imageName);
if (buildExitCode == 0)
{
break;
}
else
{
retryCount++;
if (retryCount < 3)
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
executionContext.Warning($"Docker build failed with exit code {buildExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
}
if (retryCount == 3 && buildExitCode != 0)
{
throw new InvalidOperationException($"Docker build failed with exit code {buildExitCode}");
}
foreach (var stepId in setupInfo.StepIds)
{
CachedActionContainers[stepId] = new ContainerInfo()
{
ContainerImage = imageName
};
Trace.Info($"Prepared docker image '{imageName}' for action {stepId} ({setupInfo.Container.Dockerfile})");
}
}
// git fetch --no-tags --prune --progress --no-recurse-submodules [--depth=15] origin [+refs/pull/*:refs/remote/pull/*] [+refs/tags/1:refs/tags/1]
public async Task <int> GitFetchNoTags(RunnerActionPluginExecutionContext context, string repositoryPath, string remoteName, int fetchDepth, List <string> refSpec, string additionalCommandLine, CancellationToken cancellationToken)
{
context.Debug($"Fetch git repository at: {repositoryPath} remote: {remoteName}.");
if (refSpec != null && refSpec.Count > 0)
{
refSpec = refSpec.Where(r => !string.IsNullOrEmpty(r)).ToList();
}
string options;
// If shallow fetch add --depth arg
// If the local repository is shallowed but there is no fetch depth provide for this build,
// add --unshallow to convert the shallow repository to a complete repository
if (fetchDepth > 0)
{
options = StringUtil.Format($"--no-tags --prune --progress --no-recurse-submodules --depth={fetchDepth} {remoteName} {string.Join(" ", refSpec)}");
}
else if (File.Exists(Path.Combine(repositoryPath, ".git", "shallow")))
{
options = StringUtil.Format($"--no-tags --prune --progress --no-recurse-submodules --unshallow {remoteName} {string.Join(" ", refSpec)}");
}
else
{
// default options for git fetch.
options = StringUtil.Format($"--no-tags --prune --progress --no-recurse-submodules {remoteName} {string.Join(" ", refSpec)}");
}
int retryCount = 0;
int fetchExitCode = 0;
while (retryCount < 3)
{
fetchExitCode = await ExecuteGitCommandAsync(context, repositoryPath, "fetch", options, additionalCommandLine, cancellationToken);
if (fetchExitCode == 0)
{
break;
}
else
{
if (++retryCount < 3)
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
context.Warning($"Git fetch failed with exit code {fetchExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
}
return(fetchExitCode);
}
private async Task PullActionContainerAsync(IExecutionContext executionContext, object data)
{
var setupInfo = data as ContainerSetupInfo;
ArgUtil.NotNull(setupInfo, nameof(setupInfo));
ArgUtil.NotNullOrEmpty(setupInfo.Container.Image, nameof(setupInfo.Container.Image));
executionContext.Output($"Pull down action image '{setupInfo.Container.Image}'");
// Pull down docker image with retry up to 3 times
var dockerManger = HostContext.GetService <IDockerCommandManager>();
int retryCount = 0;
int pullExitCode = 0;
while (retryCount < 3)
{
pullExitCode = await dockerManger.DockerPull(executionContext, setupInfo.Container.Image);
if (pullExitCode == 0)
{
break;
}
else
{
retryCount++;
if (retryCount < 3)
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
executionContext.Warning($"Docker pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
}
if (retryCount == 3 && pullExitCode != 0)
{
throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}");
}
foreach (var stepId in setupInfo.StepIds)
{
CachedActionContainers[stepId] = new ContainerInfo()
{
ContainerImage = setupInfo.Container.Image
};
Trace.Info($"Prepared docker image '{setupInfo.Container.Image}' for action {stepId} ({setupInfo.Container.Image})");
}
}
public async Task ConnectAsync(VssConnection jobConnection)
{
_connection = jobConnection;
int totalAttempts = 5;
int attemptCount = totalAttempts;
var configurationStore = HostContext.GetService <IConfigurationStore>();
var runnerSettings = configurationStore.GetSettings();
while (!_connection.HasAuthenticated && attemptCount-- > 0)
{
try
{
await _connection.ConnectAsync();
break;
}
catch (Exception ex) when(attemptCount > 0)
{
Trace.Info($"Catch exception during connect. {attemptCount} attempts left.");
Trace.Error(ex);
if (runnerSettings.IsHostedServer)
{
await CheckNetworkEndpointsAsync(attemptCount);
}
}
int attempt = totalAttempts - attemptCount;
TimeSpan backoff = BackoffTimerHelper.GetExponentialBackoff(attempt, TimeSpan.FromMilliseconds(100), TimeSpan.FromSeconds(3.2), TimeSpan.FromMilliseconds(100));
await Task.Delay(backoff);
}
_taskClient = _connection.GetClient <TaskHttpClient>();
_hasConnection = true;
}
public async Task <HttpResponseMessage> UploadFileAsync(
Int64 containerId,
String itemPath,
Stream fileStream,
byte[] contentId,
Int64 fileLength,
Boolean isGzipped,
Guid scopeIdentifier,
CancellationToken cancellationToken = default(CancellationToken),
int chunkSize = c_defaultChunkSize,
int chunkRetryTimes = c_defaultChunkRetryTimes,
bool uploadFirstChunk = false,
Object userState = null)
{
if (containerId < 1)
{
throw new ArgumentException(WebApiResources.ContainerIdMustBeGreaterThanZero(), "containerId");
}
if (chunkSize > c_maxChunkSize)
{
chunkSize = c_maxChunkSize;
}
// if a contentId is specified but the chunk size is not a 2mb multiple error
if (contentId != null && (chunkSize % c_ContentChunkMultiple) != 0)
{
throw new ArgumentException(FileContainerResources.ChunksizeWrongWithContentId(c_ContentChunkMultiple), "chunkSize");
}
ArgumentUtility.CheckForNull(fileStream, "fileStream");
ApiResourceVersion gzipSupportedVersion = new ApiResourceVersion(new Version(1, 0), 2);
ApiResourceVersion requestVersion = await NegotiateRequestVersionAsync(FileContainerResourceIds.FileContainer, s_currentApiVersion, userState, cancellationToken).ConfigureAwait(false);
if (isGzipped &&
(requestVersion.ApiVersion < gzipSupportedVersion.ApiVersion ||
(requestVersion.ApiVersion == gzipSupportedVersion.ApiVersion && requestVersion.ResourceVersion < gzipSupportedVersion.ResourceVersion)))
{
throw new ArgumentException(FileContainerResources.GzipNotSupportedOnServer(), "isGzipped");
}
if (isGzipped && fileStream.Length >= fileLength)
{
throw new ArgumentException(FileContainerResources.BadCompression(), "fileLength");
}
HttpRequestMessage requestMessage = null;
List <KeyValuePair <String, String> > query = AppendItemQueryString(itemPath, scopeIdentifier);
if (fileStream.Length == 0)
{
// zero byte upload
FileUploadTrace(itemPath, $"Upload zero byte file '{itemPath}'.");
requestMessage = await CreateRequestMessageAsync(HttpMethod.Put, FileContainerResourceIds.FileContainer, routeValues : new { containerId = containerId }, version : s_currentApiVersion, queryParameters : query, userState : userState, cancellationToken : cancellationToken).ConfigureAwait(false);
return(await SendAsync(requestMessage, userState, cancellationToken).ConfigureAwait(false));
}
bool multiChunk = false;
int totalChunks = 1;
if (fileStream.Length > chunkSize)
{
totalChunks = (int)Math.Ceiling(fileStream.Length / (double)chunkSize);
FileUploadTrace(itemPath, $"Begin chunking upload file '{itemPath}', chunk size '{chunkSize} Bytes', total chunks '{totalChunks}'.");
multiChunk = true;
}
else
{
FileUploadTrace(itemPath, $"File '{itemPath}' will be uploaded in one chunk.");
chunkSize = (int)fileStream.Length;
}
StreamParser streamParser = new StreamParser(fileStream, chunkSize);
SubStream currentStream = streamParser.GetNextStream();
HttpResponseMessage response = null;
Byte[] dataToSend = new Byte[chunkSize];
int currentChunk = 0;
Stopwatch uploadTimer = new Stopwatch();
while (currentStream.Length > 0 && !cancellationToken.IsCancellationRequested)
{
currentChunk++;
for (int attempt = 1; attempt <= chunkRetryTimes && !cancellationToken.IsCancellationRequested; attempt++)
{
if (attempt > 1)
{
TimeSpan backoff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(10));
FileUploadTrace(itemPath, $"Backoff {backoff.TotalSeconds} seconds before attempt '{attempt}' chunk '{currentChunk}' of file '{itemPath}'.");
await Task.Delay(backoff, cancellationToken).ConfigureAwait(false);
currentStream.Seek(0, SeekOrigin.Begin);
}
FileUploadTrace(itemPath, $"Attempt '{attempt}' for uploading chunk '{currentChunk}' of file '{itemPath}'.");
// inorder for the upload to be retryable, we need the content to be re-readable
// to ensure this we copy the chunk into a byte array and send that
// chunk size ensures we can convert the length to an int
int bytesToCopy = (int)currentStream.Length;
using (MemoryStream ms = new MemoryStream(dataToSend))
{
await currentStream.CopyToAsync(ms, bytesToCopy, cancellationToken).ConfigureAwait(false);
}
// set the content and the Content-Range header
HttpContent byteArrayContent = new ByteArrayContent(dataToSend, 0, bytesToCopy);
byteArrayContent.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/octet-stream");
byteArrayContent.Headers.ContentLength = currentStream.Length;
byteArrayContent.Headers.ContentRange = new System.Net.Http.Headers.ContentRangeHeaderValue(currentStream.StartingPostionOnOuterStream,
currentStream.EndingPostionOnOuterStream,
streamParser.Length);
FileUploadTrace(itemPath, $"Generate new HttpRequest for uploading file '{itemPath}', chunk '{currentChunk}' of '{totalChunks}'.");
try
{
if (requestMessage != null)
{
requestMessage.Dispose();
requestMessage = null;
}
requestMessage = await CreateRequestMessageAsync(
HttpMethod.Put,
FileContainerResourceIds.FileContainer,
routeValues : new { containerId = containerId },
version : s_currentApiVersion,
content : byteArrayContent,
queryParameters : query,
userState : userState,
cancellationToken : cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
{
// stop re-try on cancellation.
throw;
}
catch (Exception ex) when(attempt < chunkRetryTimes) // not the last attempt
{
FileUploadTrace(itemPath, $"Chunk '{currentChunk}' attempt '{attempt}' of file '{itemPath}' fail to create HttpRequest. Error: {ex.ToString()}.");
continue;
}
if (isGzipped)
{
//add gzip header info
byteArrayContent.Headers.ContentEncoding.Add("gzip");
byteArrayContent.Headers.Add("x-tfs-filelength", fileLength.ToString(System.Globalization.CultureInfo.InvariantCulture));
}
if (contentId != null)
{
byteArrayContent.Headers.Add("x-vso-contentId", Convert.ToBase64String(contentId)); // Base64FormattingOptions.None is default when not supplied
}
FileUploadTrace(itemPath, $"Start uploading file '{itemPath}' to server, chunk '{currentChunk}'.");
uploadTimer.Restart();
try
{
if (response != null)
{
response.Dispose();
response = null;
}
response = await SendAsync(requestMessage, userState, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
{
// stop re-try on cancellation.
throw;
}
catch (Exception ex) when(attempt < chunkRetryTimes) // not the last attempt
{
FileUploadTrace(itemPath, $"Chunk '{currentChunk}' attempt '{attempt}' of file '{itemPath}' fail to send request to server. Error: {ex.ToString()}.");
continue;
}
uploadTimer.Stop();
FileUploadTrace(itemPath, $"Finished upload chunk '{currentChunk}' of file '{itemPath}', elapsed {uploadTimer.ElapsedMilliseconds} (ms), response code '{response.StatusCode}'.");
if (multiChunk)
{
FileUploadProgress(itemPath, currentChunk, (int)Math.Ceiling(fileStream.Length / (double)chunkSize));
}
if (response.IsSuccessStatusCode)
{
break;
}
else if (IsFastFailResponse(response))
{
FileUploadTrace(itemPath, $"Chunk '{currentChunk}' attempt '{attempt}' of file '{itemPath}' received non-success status code {response.StatusCode} for sending request and cannot continue.");
break;
}
else
{
FileUploadTrace(itemPath, $"Chunk '{currentChunk}' attempt '{attempt}' of file '{itemPath}' received non-success status code {response.StatusCode} for sending request.");
continue;
}
}
// if we don't have success then bail and return the failed response
if (!response.IsSuccessStatusCode)
{
break;
}
if (contentId != null && response.StatusCode == HttpStatusCode.Created)
{
// no need to keep uploading since the server said it has all the content
FileUploadTrace(itemPath, $"Stop chunking upload the rest of the file '{itemPath}', since server already has all the content.");
break;
}
currentStream = streamParser.GetNextStream();
if (uploadFirstChunk)
{
break;
}
}
cancellationToken.ThrowIfCancellationRequested();
return(response);
}
public async Task StartContainerAsync(IExecutionContext executionContext, object data)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ContainerInfo container = data as ContainerInfo;
ArgUtil.NotNull(container, nameof(container));
ArgUtil.NotNullOrEmpty(container.ContainerImage, nameof(container.ContainerImage));
Trace.Info($"Container name: {container.ContainerName}");
Trace.Info($"Container image: {container.ContainerImage}");
Trace.Info($"Container registry: {container.ContainerRegistryEndpoint.ToString()}");
Trace.Info($"Container options: {container.ContainerCreateOptions}");
Trace.Info($"Skip container image pull: {container.SkipContainerImagePull}");
// Check whether we are inside a container.
// Our container feature requires to map working directory from host to the container.
// If we are already inside a container, we will not able to find out the real working direcotry path on the host.
#if OS_WINDOWS
// service CExecSvc is Container Execution Agent.
ServiceController[] scServices = ServiceController.GetServices();
if (scServices.Any(x => String.Equals(x.ServiceName, "cexecsvc", StringComparison.OrdinalIgnoreCase) && x.Status == ServiceControllerStatus.Running))
{
throw new NotSupportedException(StringUtil.Loc("AgentAlreadyInsideContainer"));
}
#else
var initProcessCgroup = File.ReadLines("/proc/1/cgroup");
if (initProcessCgroup.Any(x => x.IndexOf(":/docker/", StringComparison.OrdinalIgnoreCase) >= 0))
{
throw new NotSupportedException(StringUtil.Loc("AgentAlreadyInsideContainer"));
}
#endif
#if OS_WINDOWS
// Check OS version (Windows server 1803 is required)
object windowsInstallationType = Registry.GetValue(@"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion", "InstallationType", defaultValue: null);
ArgUtil.NotNull(windowsInstallationType, nameof(windowsInstallationType));
object windowsReleaseId = Registry.GetValue(@"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion", "ReleaseId", defaultValue: null);
ArgUtil.NotNull(windowsReleaseId, nameof(windowsReleaseId));
executionContext.Debug($"Current Windows version: '{windowsReleaseId} ({windowsInstallationType})'");
if (int.TryParse(windowsReleaseId.ToString(), out int releaseId))
{
if (!windowsInstallationType.ToString().StartsWith("Server", StringComparison.OrdinalIgnoreCase) || releaseId < 1803)
{
throw new NotSupportedException(StringUtil.Loc("ContainerWindowsVersionRequirement"));
}
}
else
{
throw new ArgumentOutOfRangeException(@"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\ReleaseId");
}
#endif
// Check docker client/server version
DockerVersion dockerVersion = await _dockerManger.DockerVersion(executionContext);
ArgUtil.NotNull(dockerVersion.ServerVersion, nameof(dockerVersion.ServerVersion));
ArgUtil.NotNull(dockerVersion.ClientVersion, nameof(dockerVersion.ClientVersion));
#if OS_WINDOWS
Version requiredDockerVersion = new Version(17, 6);
#else
Version requiredDockerVersion = new Version(17, 12);
#endif
if (dockerVersion.ServerVersion < requiredDockerVersion)
{
throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerServerVersion", requiredDockerVersion, _dockerManger.DockerPath, dockerVersion.ServerVersion));
}
if (dockerVersion.ClientVersion < requiredDockerVersion)
{
throw new NotSupportedException(StringUtil.Loc("MinRequiredDockerClientVersion", requiredDockerVersion, _dockerManger.DockerPath, dockerVersion.ClientVersion));
}
// Login to private docker registry
string registryServer = string.Empty;
if (container.ContainerRegistryEndpoint != Guid.Empty)
{
var registryEndpoint = executionContext.Endpoints.FirstOrDefault(x => x.Type == "dockerregistry" && x.Id == container.ContainerRegistryEndpoint);
ArgUtil.NotNull(registryEndpoint, nameof(registryEndpoint));
string username = string.Empty;
string password = string.Empty;
registryEndpoint.Authorization?.Parameters?.TryGetValue("registry", out registryServer);
registryEndpoint.Authorization?.Parameters?.TryGetValue("username", out username);
registryEndpoint.Authorization?.Parameters?.TryGetValue("password", out password);
ArgUtil.NotNullOrEmpty(registryServer, nameof(registryServer));
ArgUtil.NotNullOrEmpty(username, nameof(username));
ArgUtil.NotNullOrEmpty(password, nameof(password));
int loginExitCode = await _dockerManger.DockerLogin(executionContext, registryServer, username, password);
if (loginExitCode != 0)
{
throw new InvalidOperationException($"Docker login fail with exit code {loginExitCode}");
}
}
try
{
if (!container.SkipContainerImagePull)
{
if (!string.IsNullOrEmpty(registryServer) &&
registryServer.IndexOf("index.docker.io", StringComparison.OrdinalIgnoreCase) < 0)
{
var registryServerUri = new Uri(registryServer);
if (!container.ContainerImage.StartsWith(registryServerUri.Authority, StringComparison.OrdinalIgnoreCase))
{
container.ContainerImage = $"{registryServerUri.Authority}/{container.ContainerImage}";
}
}
// Pull down docker image with retry up to 3 times
int retryCount = 0;
int pullExitCode = 0;
while (retryCount < 3)
{
pullExitCode = await _dockerManger.DockerPull(executionContext, container.ContainerImage);
if (pullExitCode == 0)
{
break;
}
else
{
retryCount++;
if (retryCount < 3)
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
executionContext.Warning($"Docker pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
}
if (retryCount == 3 && pullExitCode != 0)
{
throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}");
}
}
// Mount folder into container
#if OS_WINDOWS
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Work), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Work))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools))));
#else
string workingDirectory = Path.GetDirectoryName(executionContext.Variables.Get(Constants.Variables.System.DefaultWorkingDirectory).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar));
container.MountVolumes.Add(new MountVolume(container.TranslateToHostPath(workingDirectory), workingDirectory));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Temp), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Temp))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tasks), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tasks))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)), true));
// Ensure .taskkey file exist so we can mount it.
string taskKeyFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), ".taskkey");
if (!File.Exists(taskKeyFile))
{
File.WriteAllText(taskKeyFile, string.Empty);
}
container.MountVolumes.Add(new MountVolume(taskKeyFile, container.TranslateToContainerPath(taskKeyFile)));
#endif
#if !OS_WINDOWS
if (string.IsNullOrEmpty(container.ContainerNetwork)) // create network when Windows support it.
{
// Create local docker network for this job to avoid port conflict when multiple agents run on same machine.
container.ContainerNetwork = $"vsts_network_{Guid.NewGuid().ToString("N")}";
int networkExitCode = await _dockerManger.DockerNetworkCreate(executionContext, container.ContainerNetwork);
if (networkExitCode != 0)
{
throw new InvalidOperationException($"Docker network create fail with exit code {networkExitCode}");
}
// Expose docker network to env
executionContext.Variables.Set(Constants.Variables.Agent.ContainerNetwork, container.ContainerNetwork);
}
#endif
container.ContainerId = await _dockerManger.DockerCreate(context : executionContext,
displayName : container.ContainerDisplayName,
image : container.ContainerImage,
mountVolumes : container.MountVolumes,
network : container.ContainerNetwork,
options : container.ContainerCreateOptions,
environment : container.ContainerEnvironmentVariables);
ArgUtil.NotNullOrEmpty(container.ContainerId, nameof(container.ContainerId));
executionContext.Variables.Set(Constants.Variables.Agent.ContainerId, container.ContainerId);
// Start container
int startExitCode = await _dockerManger.DockerStart(executionContext, container.ContainerId);
if (startExitCode != 0)
{
throw new InvalidOperationException($"Docker start fail with exit code {startExitCode}");
}
}
finally
{
// Logout for private registry
if (!string.IsNullOrEmpty(registryServer))
{
int logoutExitCode = await _dockerManger.DockerLogout(executionContext, registryServer);
if (logoutExitCode != 0)
{
executionContext.Error($"Docker logout fail with exit code {logoutExitCode}");
}
}
}
#if !OS_WINDOWS
// Ensure bash exist in the image
int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"which bash");
if (execWhichBashExitCode != 0)
{
try
{
// Make sure container is up and running
var psOutputs = await _dockerManger.DockerPS(executionContext, container.ContainerId, "--filter status=running");
if (psOutputs.FirstOrDefault(x => !string.IsNullOrEmpty(x))?.StartsWith(container.ContainerId) != true)
{
// container is not up and running, pull docker log for this container.
await _dockerManger.DockerPS(executionContext, container.ContainerId, string.Empty);
int logsExitCode = await _dockerManger.DockerLogs(executionContext, container.ContainerId);
if (logsExitCode != 0)
{
executionContext.Warning($"Docker logs fail with exit code {logsExitCode}");
}
executionContext.Warning($"Docker container {container.ContainerId} is not in running state.");
}
}
catch (Exception ex)
{
// pull container log is best effort.
Trace.Error("Catch exception when check container log and container status.");
Trace.Error(ex);
}
throw new InvalidOperationException($"Docker exec fail with exit code {execWhichBashExitCode}");
}
// Get current username
container.CurrentUserName = (await ExecuteCommandAsync(executionContext, "whoami", string.Empty)).FirstOrDefault();
ArgUtil.NotNullOrEmpty(container.CurrentUserName, nameof(container.CurrentUserName));
// Get current userId
container.CurrentUserId = (await ExecuteCommandAsync(executionContext, "id", $"-u {container.CurrentUserName}")).FirstOrDefault();
ArgUtil.NotNullOrEmpty(container.CurrentUserId, nameof(container.CurrentUserId));
executionContext.Output(StringUtil.Loc("CreateUserWithSameUIDInsideContainer", container.CurrentUserId));
// Create an user with same uid as the agent run as user inside the container.
// All command execute in docker will run as Root by default,
// this will cause the agent on the host machine doesn't have permission to any new file/folder created inside the container.
// So, we create a user account with same UID inside the container and let all docker exec command run as that user.
string containerUserName = string.Empty;
// We need to find out whether there is a user with same UID inside the container
List <string> userNames = new List <string>();
int execGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"grep {container.CurrentUserId} /etc/passwd | cut -f1 -d:\"", userNames);
if (execGrepExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGrepExitCode}");
}
if (userNames.Count > 0)
{
// check all potential username that might match the UID.
foreach (string username in userNames)
{
int execIdExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"id -u {username}");
if (execIdExitCode == 0)
{
containerUserName = username;
break;
}
}
}
// Create a new user with same UID
if (string.IsNullOrEmpty(containerUserName))
{
containerUserName = $"{container.CurrentUserName}_VSTSContainer";
int execUseraddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"useradd -m -u {container.CurrentUserId} {containerUserName}");
if (execUseraddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execUseraddExitCode}");
}
}
executionContext.Output(StringUtil.Loc("GrantContainerUserSUDOPrivilege", containerUserName));
// Create a new vsts_sudo group for giving sudo permission
int execGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd VSTS_Container_SUDO");
if (execGroupaddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGroupaddExitCode}");
}
// Add the new created user to the new created VSTS_SUDO group.
int execUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G VSTS_Container_SUDO {containerUserName}");
if (execUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execUsermodExitCode}");
}
// Allow the new vsts_sudo group run any sudo command without providing password.
int execEchoExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"su -c \"echo '%VSTS_Container_SUDO ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers\"");
if (execUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execEchoExitCode}");
}
#endif
}
private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, Pipelines.ActionStep repositoryAction)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
var repositoryReference = repositoryAction.Reference as Pipelines.RepositoryPathReference;
ArgUtil.NotNull(repositoryReference, nameof(repositoryReference));
if (string.Equals(repositoryReference.RepositoryType, Pipelines.PipelineConstants.SelfAlias, StringComparison.OrdinalIgnoreCase))
{
Trace.Info($"Repository action is in 'self' repository.");
return;
}
if (!string.Equals(repositoryReference.RepositoryType, Pipelines.RepositoryTypes.GitHub, StringComparison.OrdinalIgnoreCase))
{
throw new NotSupportedException(repositoryReference.RepositoryType);
}
ArgUtil.NotNullOrEmpty(repositoryReference.Name, nameof(repositoryReference.Name));
ArgUtil.NotNullOrEmpty(repositoryReference.Ref, nameof(repositoryReference.Ref));
string destDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), repositoryReference.Name.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), repositoryReference.Ref);
if (File.Exists(destDirectory + ".completed"))
{
executionContext.Debug($"Action '{repositoryReference.Name}@{repositoryReference.Ref}' already downloaded at '{destDirectory}'.");
return;
}
else
{
// make sure we get a clean folder ready to use.
IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken);
Directory.CreateDirectory(destDirectory);
executionContext.Output($"Download action repository '{repositoryReference.Name}@{repositoryReference.Ref}'");
}
#if OS_WINDOWS
string archiveLink = $"https://api.github.com/repos/{repositoryReference.Name}/zipball/{repositoryReference.Ref}";
#else
string archiveLink = $"https://api.github.com/repos/{repositoryReference.Name}/tarball/{repositoryReference.Ref}";
#endif
Trace.Info($"Download archive '{archiveLink}' to '{destDirectory}'.");
//download and extract action in a temp folder and rename it on success
string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), "_temp_" + Guid.NewGuid());
Directory.CreateDirectory(tempDirectory);
#if OS_WINDOWS
string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.zip");
#else
string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.tar.gz");
#endif
Trace.Info($"Save archive '{archiveLink}' into {archiveFile}.");
try
{
int retryCount = 0;
// Allow up to 20 * 60s for any action to be downloaded from github graph.
int timeoutSeconds = 20 * 60;
while (retryCount < 3)
{
using (var actionDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
using (var actionDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(actionDownloadTimeout.Token, executionContext.CancellationToken))
{
try
{
//open zip stream in async mode
using (FileStream fs = new FileStream(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true))
using (var httpClientHandler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(httpClientHandler))
{
var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN");
if (string.IsNullOrEmpty(authToken))
{
// TODO: Depreciate the PREVIEW_ACTION_TOKEN
authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN");
}
if (!string.IsNullOrEmpty(authToken))
{
HostContext.SecretMasker.AddValue(authToken);
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"PAT:{authToken}"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
}
else
{
var accessToken = executionContext.GetGitHubContext("token");
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
}
httpClient.DefaultRequestHeaders.UserAgent.Add(HostContext.UserAgent);
using (var result = await httpClient.GetStreamAsync(archiveLink))
{
await result.CopyToAsync(fs, _defaultCopyBufferSize, actionDownloadCancellation.Token);
await fs.FlushAsync(actionDownloadCancellation.Token);
// download succeed, break out the retry loop.
break;
}
}
}
catch (OperationCanceledException) when(executionContext.CancellationToken.IsCancellationRequested)
{
Trace.Info($"Action download has been cancelled.");
throw;
}
catch (Exception ex) when(retryCount < 2)
{
retryCount++;
Trace.Error($"Fail to download archive '{archiveLink}' -- Attempt: {retryCount}");
Trace.Error(ex);
if (actionDownloadTimeout.Token.IsCancellationRequested)
{
// action download didn't finish within timeout
executionContext.Warning($"Action '{archiveLink}' didn't finish download within {timeoutSeconds} seconds.");
}
else
{
executionContext.Warning($"Failed to download action '{archiveLink}'. Error {ex.Message}");
}
}
}
if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("_GITHUB_ACTION_DOWNLOAD_NO_BACKOFF")))
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30));
executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
ArgUtil.NotNullOrEmpty(archiveFile, nameof(archiveFile));
executionContext.Debug($"Download '{archiveLink}' to '{archiveFile}'");
var stagingDirectory = Path.Combine(tempDirectory, "_staging");
Directory.CreateDirectory(stagingDirectory);
#if OS_WINDOWS
ZipFile.ExtractToDirectory(archiveFile, stagingDirectory);
#else
string tar = WhichUtil.Which("tar", require: true, trace: Trace);
// tar -xzf
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Info(args.Data);
}
});
processInvoker.ErrorDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Error(args.Data);
}
});
int exitCode = await processInvoker.ExecuteAsync(stagingDirectory, tar, $"-xzf \"{archiveFile}\"", null, executionContext.CancellationToken);
if (exitCode != 0)
{
throw new NotSupportedException($"Can't use 'tar -xzf' extract archive file: {archiveFile}. return code: {exitCode}.");
}
}
#endif
// repository archive from github always contains a nested folder
var subDirectories = new DirectoryInfo(stagingDirectory).GetDirectories();
if (subDirectories.Length != 1)
{
throw new InvalidOperationException($"'{archiveFile}' contains '{subDirectories.Length}' directories");
}
else
{
executionContext.Debug($"Unwrap '{subDirectories[0].Name}' to '{destDirectory}'");
IOUtil.CopyDirectory(subDirectories[0].FullName, destDirectory, executionContext.CancellationToken);
}
Trace.Verbose("Create watermark file indicate action download succeed.");
File.WriteAllText(destDirectory + ".completed", DateTime.UtcNow.ToString());
executionContext.Debug($"Archive '{archiveFile}' has been unzipped into '{destDirectory}'.");
Trace.Info("Finished getting action repository.");
}
finally
{
try
{
//if the temp folder wasn't moved -> wipe it
if (Directory.Exists(tempDirectory))
{
Trace.Verbose("Deleting action temp folder: {0}", tempDirectory);
IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast.
}
}
catch (Exception ex)
{
//it is not critical if we fail to delete the temp folder
Trace.Warning("Failed to delete temp folder '{0}'. Exception: {1}", tempDirectory, ex);
}
}
}
public async Task DownloadFromContainerAsync(
RunnerActionPluginExecutionContext context,
String destination,
CancellationToken cancellationToken)
{
// Find out all container items need to be processed
List <FileContainerItem> containerItems = new List <FileContainerItem>();
int retryCount = 0;
while (retryCount < 3)
{
try
{
containerItems = await _fileContainerHttpClient.QueryContainerItemsAsync(_containerId,
_projectId,
_containerPath,
cancellationToken : cancellationToken);
break;
}
catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
{
context.Debug($"Container query has been cancelled.");
throw;
}
catch (Exception ex) when(retryCount < 2)
{
retryCount++;
context.Warning($"Fail to query container items under #/{_containerId}/{_containerPath}, Error: {ex.Message}");
context.Debug(ex.ToString());
}
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
context.Warning($"Back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
if (containerItems.Count == 0)
{
context.Output($"There is nothing under #/{_containerId}/{_containerPath}");
return;
}
// container items will include both folders, files and even file with zero size
// Create all required empty folders and emptry files, gather a list of files that we need to download from server.
int foldersCreated = 0;
int emptryFilesCreated = 0;
List <DownloadInfo> downloadFiles = new List <DownloadInfo>();
foreach (var item in containerItems.OrderBy(x => x.Path))
{
if (!item.Path.StartsWith(_containerPath, StringComparison.OrdinalIgnoreCase))
{
throw new ArgumentOutOfRangeException($"Item {item.Path} is not under #/{_containerId}/{_containerPath}");
}
var localRelativePath = item.Path.Substring(_containerPath.Length).TrimStart('/');
var localPath = Path.Combine(destination, localRelativePath);
if (item.ItemType == ContainerItemType.Folder)
{
context.Debug($"Ensure folder exists: {localPath}");
Directory.CreateDirectory(localPath);
foldersCreated++;
}
else if (item.ItemType == ContainerItemType.File)
{
if (item.FileLength == 0)
{
context.Debug($"Create empty file at: {localPath}");
var parentDirectory = Path.GetDirectoryName(localPath);
Directory.CreateDirectory(parentDirectory);
IOUtil.DeleteFile(localPath);
using (new FileStream(localPath, FileMode.Create))
{
}
emptryFilesCreated++;
}
else
{
context.Debug($"Prepare download {item.Path} to {localPath}");
downloadFiles.Add(new DownloadInfo(item.Path, localPath));
}
}
else
{
throw new NotSupportedException(item.ItemType.ToString());
}
}
if (foldersCreated > 0)
{
context.Output($"{foldersCreated} folders created.");
}
if (emptryFilesCreated > 0)
{
context.Output($"{emptryFilesCreated} empty files created.");
}
if (downloadFiles.Count == 0)
{
context.Output($"There is nothing to download");
return;
}
// Start multi-task to download all files.
using (_downloadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
// try download all files for the first time.
DownloadResult downloadResult = await ParallelDownloadAsync(context, downloadFiles.AsReadOnly(), Math.Min(downloadFiles.Count, Environment.ProcessorCount), _downloadCancellationTokenSource.Token);
if (downloadResult.FailedFiles.Count == 0)
{
// all files have been download succeed.
context.Output($"{downloadFiles.Count} files download succeed.");
return;
}
else
{
context.Output($"{downloadResult.FailedFiles.Count} files failed to download, retry these files after a minute.");
}
// Delay 1 min then retry failed files.
for (int timer = 60; timer > 0; timer -= 5)
{
context.Output($"Retry file download after {timer} seconds.");
await Task.Delay(TimeSpan.FromSeconds(5), _uploadCancellationTokenSource.Token);
}
// Retry download all failed files.
context.Output($"Start retry {downloadResult.FailedFiles.Count} failed files upload.");
DownloadResult retryDownloadResult = await ParallelDownloadAsync(context, downloadResult.FailedFiles.AsReadOnly(), Math.Min(downloadResult.FailedFiles.Count, Environment.ProcessorCount), _downloadCancellationTokenSource.Token);
if (retryDownloadResult.FailedFiles.Count == 0)
{
// all files have been download succeed after retry.
context.Output($"{downloadResult.FailedFiles} files download succeed after retry.");
return;
}
else
{
throw new Exception($"{retryDownloadResult.FailedFiles.Count} files failed to download even after retry.");
}
}
}
private async Task DownloadAsync(IExecutionContext executionContext, Pipelines.TaskStepDefinitionReference task)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(task, nameof(task));
ArgUtil.NotNullOrEmpty(task.Version, nameof(task.Version));
var taskServer = HostContext.GetService <ITaskServer>();
// first check to see if we already have the task
string destDirectory = GetDirectory(task);
Trace.Info($"Ensuring task exists: ID '{task.Id}', version '{task.Version}', name '{task.Name}', directory '{destDirectory}'.");
var configurationStore = HostContext.GetService <IConfigurationStore>();
AgentSettings settings = configurationStore.GetSettings();
Boolean signingEnabled = !String.IsNullOrEmpty(settings.Fingerprint);
if (File.Exists(destDirectory + ".completed") && !signingEnabled)
{
executionContext.Debug($"Task '{task.Name}' already downloaded at '{destDirectory}'.");
return;
}
String taskZipPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.TaskZips), $"{task.Name}_{task.Id}_{task.Version}.zip");
if (signingEnabled && File.Exists(taskZipPath))
{
executionContext.Debug($"Task '{task.Name}' already downloaded at '{taskZipPath}'.");
// We need to extract the zip now because the task.json metadata for the task is used in JobExtension.InitializeJob.
// This is fine because we overwrite the contents at task run time.
if (!File.Exists(destDirectory + ".completed"))
{
// The zip exists but it hasn't been extracted yet.
IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken);
ExtractZip(taskZipPath, destDirectory);
}
return;
}
// delete existing task folder.
Trace.Verbose("Deleting task destination folder: {0}", destDirectory);
IOUtil.DeleteDirectory(destDirectory, CancellationToken.None);
// Inform the user that a download is taking place. The download could take a while if
// the task zip is large. It would be nice to print the localized name, but it is not
// available from the reference included in the job message.
executionContext.Output(StringUtil.Loc("DownloadingTask0", task.Name, task.Version));
string zipFile = string.Empty;
var version = new TaskVersion(task.Version);
//download and extract task in a temp folder and rename it on success
string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Tasks), "_temp_" + Guid.NewGuid());
try
{
Directory.CreateDirectory(tempDirectory);
int retryCount = 0;
// Allow up to 20 * 60s for any task to be downloaded from service.
// Base on Kusto, the longest we have on the service today is over 850 seconds.
// Timeout limit can be overwrite by environment variable
if (!int.TryParse(Environment.GetEnvironmentVariable("VSTS_TASK_DOWNLOAD_TIMEOUT") ?? string.Empty, out int timeoutSeconds))
{
timeoutSeconds = 20 * 60;
}
while (retryCount < 3)
{
using (var taskDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
using (var taskDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(taskDownloadTimeout.Token, executionContext.CancellationToken))
{
try
{
zipFile = Path.Combine(tempDirectory, string.Format("{0}.zip", Guid.NewGuid()));
//open zip stream in async mode
using (FileStream fs = new FileStream(zipFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true))
using (Stream result = await taskServer.GetTaskContentZipAsync(task.Id, version, taskDownloadCancellation.Token))
{
await result.CopyToAsync(fs, _defaultCopyBufferSize, taskDownloadCancellation.Token);
await fs.FlushAsync(taskDownloadCancellation.Token);
// download succeed, break out the retry loop.
break;
}
}
catch (OperationCanceledException) when(executionContext.CancellationToken.IsCancellationRequested)
{
Trace.Info($"Task download has been cancelled.");
throw;
}
catch (Exception ex) when(retryCount < 2)
{
retryCount++;
Trace.Error($"Fail to download task '{task.Id} ({task.Name}/{task.Version})' -- Attempt: {retryCount}");
Trace.Error(ex);
if (taskDownloadTimeout.Token.IsCancellationRequested)
{
// task download didn't finish within timeout
executionContext.Warning(StringUtil.Loc("TaskDownloadTimeout", task.Name, timeoutSeconds));
}
else
{
executionContext.Warning(StringUtil.Loc("TaskDownloadFailed", task.Name, ex.Message));
if (ex.InnerException != null)
{
executionContext.Warning("Inner Exception: {ex.InnerException.Message}");
}
}
}
}
if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("VSTS_TASK_DOWNLOAD_NO_BACKOFF")))
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30));
executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
if (signingEnabled)
{
Directory.CreateDirectory(HostContext.GetDirectory(WellKnownDirectory.TaskZips));
// Copy downloaded zip to the cache on disk for future extraction.
executionContext.Debug($"Copying from {zipFile} to {taskZipPath}");
File.Copy(zipFile, taskZipPath);
}
// We need to extract the zip regardless of whether or not signing is enabled because the task.json metadata for the task is used in JobExtension.InitializeJob.
// This is fine because we overwrite the contents at task run time.
Directory.CreateDirectory(destDirectory);
ExtractZip(zipFile, destDirectory);
executionContext.Debug($"Task '{task.Name}' has been downloaded into '{(signingEnabled ? taskZipPath : destDirectory)}'.");
Trace.Info("Finished getting task.");
}
finally
{
try
{
//if the temp folder wasn't moved -> wipe it
if (Directory.Exists(tempDirectory))
{
Trace.Verbose("Deleting task temp folder: {0}", tempDirectory);
IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast.
}
}
catch (Exception ex)
{
//it is not critical if we fail to delete the temp folder
Trace.Warning("Failed to delete temp folder '{0}'. Exception: {1}", tempDirectory, ex);
executionContext.Warning(StringUtil.Loc("FailedDeletingTempDirectory0Message1", tempDirectory, ex.Message));
}
}
}
public async Task <TaskAgentMessage> GetNextMessageAsync(CancellationToken token)
{
Trace.Entering();
ArgUtil.NotNull(_session, nameof(_session));
ArgUtil.NotNull(_settings, nameof(_settings));
bool encounteringError = false;
int continuousError = 0;
string errorMessage = string.Empty;
Stopwatch heartbeat = new Stopwatch();
heartbeat.Restart();
while (true)
{
token.ThrowIfCancellationRequested();
TaskAgentMessage message = null;
try
{
message = await _runnerServer.GetAgentMessageAsync(_settings.PoolId,
_session.SessionId,
_lastMessageId,
token);
// Decrypt the message body if the session is using encryption
message = DecryptMessage(message);
if (message != null)
{
_lastMessageId = message.MessageId;
}
if (encounteringError) //print the message once only if there was an error
{
_term.WriteLine($"{DateTime.UtcNow:u}: Runner reconnected.");
encounteringError = false;
continuousError = 0;
}
if (_needToCheckAuthorizationUrlUpdate &&
_authorizationUrlMigrationBackgroundTask?.IsCompleted == true)
{
if (HostContext.GetService <IJobDispatcher>().Busy ||
HostContext.GetService <ISelfUpdater>().Busy)
{
Trace.Info("Job or runner updates in progress, update credentials next time.");
}
else
{
try
{
var newCred = await _authorizationUrlMigrationBackgroundTask;
await _runnerServer.ConnectAsync(new Uri(_settings.ServerUrl), newCred);
Trace.Info("Updated connection to use migrated credential for next GetMessage call.");
_useMigratedCredentials = true;
_authorizationUrlMigrationBackgroundTask = null;
_needToCheckAuthorizationUrlUpdate = false;
}
catch (Exception ex)
{
Trace.Error("Fail to refresh connection with new authorization url.");
Trace.Error(ex);
}
}
}
if (_authorizationUrlRollbackReattemptDelayBackgroundTask?.IsCompleted == true)
{
try
{
// we rolled back to use original creds about 2 days before, now it's a good time to try migrated creds again.
Trace.Info("Re-attempt to use migrated credential");
var migratedCreds = _credMgr.LoadCredentials();
await _runnerServer.ConnectAsync(new Uri(_settings.ServerUrl), migratedCreds);
_useMigratedCredentials = true;
_authorizationUrlRollbackReattemptDelayBackgroundTask = null;
}
catch (Exception ex)
{
Trace.Error("Fail to refresh connection with new authorization url on rollback reattempt.");
Trace.Error(ex);
}
}
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
Trace.Info("Get next message has been cancelled.");
throw;
}
catch (TaskAgentAccessTokenExpiredException)
{
Trace.Info("Runner OAuth token has been revoked. Unable to pull message.");
throw;
}
catch (Exception ex)
{
Trace.Error("Catch exception during get next message.");
Trace.Error(ex);
// don't retry if SkipSessionRecover = true, DT service will delete agent session to stop agent from taking more jobs.
if (ex is TaskAgentSessionExpiredException && !_settings.SkipSessionRecover && await CreateSessionAsync(token))
{
Trace.Info($"{nameof(TaskAgentSessionExpiredException)} received, recovered by recreate session.");
}
else if (!IsGetNextMessageExceptionRetriable(ex))
{
if (_useMigratedCredentials)
{
// migrated credentials might cause lose permission during permission check,
// we will force to use original credential and try again
_useMigratedCredentials = false;
var reattemptBackoff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromHours(24), TimeSpan.FromHours(36));
_authorizationUrlRollbackReattemptDelayBackgroundTask = HostContext.Delay(reattemptBackoff, token); // retry migrated creds in 24-36 hours.
var originalCreds = _credMgr.LoadCredentials(false);
await _runnerServer.ConnectAsync(new Uri(_settings.ServerUrl), originalCreds);
Trace.Error("Fallback to original credentials and try again.");
}
else
{
throw;
}
}
else
{
continuousError++;
//retry after a random backoff to avoid service throttling
//in case of there is a service error happened and all agents get kicked off of the long poll and all agent try to reconnect back at the same time.
if (continuousError <= 5)
{
// random backoff [15, 30]
_getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30), _getNextMessageRetryInterval);
}
else
{
// more aggressive backoff [30, 60]
_getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(60), _getNextMessageRetryInterval);
}
if (!encounteringError)
{
//print error only on the first consecutive error
_term.WriteError($"{DateTime.UtcNow:u}: Runner connect error: {ex.Message}. Retrying until reconnected.");
encounteringError = true;
}
// re-create VssConnection before next retry
await _runnerServer.RefreshConnectionAsync(RunnerConnectionType.MessageQueue, TimeSpan.FromSeconds(60));
Trace.Info("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds);
await HostContext.Delay(_getNextMessageRetryInterval, token);
}
}
if (message == null)
{
if (heartbeat.Elapsed > TimeSpan.FromMinutes(30))
{
Trace.Info($"No message retrieved from session '{_session.SessionId}' within last 30 minutes.");
heartbeat.Restart();
}
else
{
Trace.Verbose($"No message retrieved from session '{_session.SessionId}'.");
}
continue;
}
Trace.Info($"Message '{message.MessageId}' received from session '{_session.SessionId}'.");
return(message);
}
}
private async Task StartContainerAsync(IExecutionContext executionContext, ContainerInfo container)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(container, nameof(container));
ArgUtil.NotNullOrEmpty(container.ContainerImage, nameof(container.ContainerImage));
Trace.Info($"Container name: {container.ContainerName}");
Trace.Info($"Container image: {container.ContainerImage}");
Trace.Info($"Container options: {container.ContainerCreateOptions}");
var groupName = container.IsJobContainer ? "Starting job container" : $"Starting {container.ContainerNetworkAlias} service container";
executionContext.Output($"##[group]{groupName}");
foreach (var port in container.UserPortMappings)
{
Trace.Info($"User provided port: {port.Value}");
}
foreach (var volume in container.UserMountVolumes)
{
Trace.Info($"User provided volume: {volume.Value}");
var mount = new MountVolume(volume.Value);
if (string.Equals(mount.SourceVolumePath, "/", StringComparison.OrdinalIgnoreCase))
{
executionContext.Warning($"Volume mount {volume.Value} is going to mount '/' into the container which may cause file ownership change in the entire file system and cause Actions Runner to lose permission to access the disk.");
}
}
// Pull down docker image with retry up to 3 times
int retryCount = 0;
int pullExitCode = 0;
while (retryCount < 3)
{
pullExitCode = await _dockerManger.DockerPull(executionContext, container.ContainerImage);
if (pullExitCode == 0)
{
break;
}
else
{
retryCount++;
if (retryCount < 3)
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
executionContext.Warning($"Docker pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
}
if (retryCount == 3 && pullExitCode != 0)
{
throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}");
}
if (container.IsJobContainer)
{
// Configure job container - Mount workspace and tools, set up environment, and start long running process
var githubContext = executionContext.ExpressionValues["github"] as GitHubContext;
ArgUtil.NotNull(githubContext, nameof(githubContext));
var workingDirectory = githubContext["workspace"] as StringContextData;
ArgUtil.NotNullOrEmpty(workingDirectory, nameof(workingDirectory));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Work), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Work))));
#if OS_WINDOWS
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals))));
#else
if (Environment.GetEnvironmentVariable("K8S_POD_NAME") != null)
{
container.MountVolumes.Add(new MountVolume(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), "__externals_copy"), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)), true));
}
else
{
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)), true));
}
#endif
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Temp), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Temp))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Actions), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Actions))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools))));
var tempHomeDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Temp), "_github_home");
Directory.CreateDirectory(tempHomeDirectory);
container.MountVolumes.Add(new MountVolume(tempHomeDirectory, "/github/home"));
container.AddPathTranslateMapping(tempHomeDirectory, "/github/home");
container.ContainerEnvironmentVariables["HOME"] = container.TranslateToContainerPath(tempHomeDirectory);
var tempWorkflowDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Temp), "_github_workflow");
Directory.CreateDirectory(tempWorkflowDirectory);
container.MountVolumes.Add(new MountVolume(tempWorkflowDirectory, "/github/workflow"));
container.AddPathTranslateMapping(tempWorkflowDirectory, "/github/workflow");
container.ContainerWorkDirectory = container.TranslateToContainerPath(workingDirectory);
container.ContainerEntryPoint = "tail";
container.ContainerEntryPointArgs = "\"-f\" \"/dev/null\"";
}
container.ContainerId = await _dockerManger.DockerCreate(executionContext, container);
ArgUtil.NotNullOrEmpty(container.ContainerId, nameof(container.ContainerId));
// Start container
int startExitCode = await _dockerManger.DockerStart(executionContext, container.ContainerId);
if (startExitCode != 0)
{
throw new InvalidOperationException($"Docker start fail with exit code {startExitCode}");
}
try
{
// Make sure container is up and running
var psOutputs = await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --filter status=running --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\"");
if (psOutputs.FirstOrDefault(x => !string.IsNullOrEmpty(x))?.StartsWith(container.ContainerId) != true)
{
// container is not up and running, pull docker log for this container.
await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\"");
int logsExitCode = await _dockerManger.DockerLogs(executionContext, container.ContainerId);
if (logsExitCode != 0)
{
executionContext.Warning($"Docker logs fail with exit code {logsExitCode}");
}
executionContext.Warning($"Docker container {container.ContainerId} is not in running state.");
}
}
catch (Exception ex)
{
// pull container log is best effort.
Trace.Error("Catch exception when check container log and container status.");
Trace.Error(ex);
}
// Gather runtime container information
if (!container.IsJobContainer)
{
var service = new DictionaryContextData()
{
["id"] = new StringContextData(container.ContainerId),
["ports"] = new DictionaryContextData(),
["network"] = new StringContextData(container.ContainerNetwork)
};
container.AddPortMappings(await _dockerManger.DockerPort(executionContext, container.ContainerId));
foreach (var port in container.PortMappings)
{
(service["ports"] as DictionaryContextData)[port.ContainerPort] = new StringContextData(port.HostPort);
}
executionContext.JobContext.Services[container.ContainerNetworkAlias] = service;
}
else
{
var configEnvFormat = "--format \"{{range .Config.Env}}{{println .}}{{end}}\"";
var containerEnv = await _dockerManger.DockerInspect(executionContext, container.ContainerId, configEnvFormat);
container.ContainerRuntimePath = DockerUtil.ParsePathFromConfigEnv(containerEnv);
executionContext.JobContext.Container["id"] = new StringContextData(container.ContainerId);
}
executionContext.Output("##[endgroup]");
}
private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, ActionDownloadDetails actionDownloadDetails, string destDirectory)
{
//download and extract action in a temp folder and rename it on success
string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), "_temp_" + Guid.NewGuid());
Directory.CreateDirectory(tempDirectory);
#if OS_WINDOWS
string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.zip");
#else
string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.tar.gz");
#endif
string link = actionDownloadDetails.ArchiveLink;
Trace.Info($"Save archive '{link}' into {archiveFile}.");
try
{
int retryCount = 0;
// Allow up to 20 * 60s for any action to be downloaded from github graph.
int timeoutSeconds = 20 * 60;
while (retryCount < 3)
{
using (var actionDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
using (var actionDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(actionDownloadTimeout.Token, executionContext.CancellationToken))
{
try
{
//open zip stream in async mode
using (FileStream fs = new FileStream(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true))
using (var httpClientHandler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(httpClientHandler))
{
actionDownloadDetails.ConfigureAuthorization(executionContext, httpClient);
httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents);
using (var response = await httpClient.GetAsync(link))
{
if (response.IsSuccessStatusCode)
{
using (var result = await response.Content.ReadAsStreamAsync())
{
await result.CopyToAsync(fs, _defaultCopyBufferSize, actionDownloadCancellation.Token);
await fs.FlushAsync(actionDownloadCancellation.Token);
// download succeed, break out the retry loop.
break;
}
}
else if (response.StatusCode == HttpStatusCode.NotFound)
{
// It doesn't make sense to retry in this case, so just stop
throw new ActionNotFoundException(new Uri(link));
}
else
{
// Something else bad happened, let's go to our retry logic
response.EnsureSuccessStatusCode();
}
}
}
}
catch (OperationCanceledException) when(executionContext.CancellationToken.IsCancellationRequested)
{
Trace.Info("Action download has been cancelled.");
throw;
}
catch (ActionNotFoundException)
{
Trace.Info($"The action at '{link}' does not exist");
throw;
}
catch (Exception ex) when(retryCount < 2)
{
retryCount++;
Trace.Error($"Fail to download archive '{link}' -- Attempt: {retryCount}");
Trace.Error(ex);
if (actionDownloadTimeout.Token.IsCancellationRequested)
{
// action download didn't finish within timeout
executionContext.Warning($"Action '{link}' didn't finish download within {timeoutSeconds} seconds.");
}
else
{
executionContext.Warning($"Failed to download action '{link}'. Error: {ex.Message}");
}
}
}
if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("_GITHUB_ACTION_DOWNLOAD_NO_BACKOFF")))
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30));
executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
ArgUtil.NotNullOrEmpty(archiveFile, nameof(archiveFile));
executionContext.Debug($"Download '{link}' to '{archiveFile}'");
var stagingDirectory = Path.Combine(tempDirectory, "_staging");
Directory.CreateDirectory(stagingDirectory);
#if OS_WINDOWS
ZipFile.ExtractToDirectory(archiveFile, stagingDirectory);
#else
string tar = WhichUtil.Which("tar", require: true, trace: Trace);
// tar -xzf
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Info(args.Data);
}
});
processInvoker.ErrorDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Error(args.Data);
}
});
int exitCode = await processInvoker.ExecuteAsync(stagingDirectory, tar, $"-xzf \"{archiveFile}\"", null, executionContext.CancellationToken);
if (exitCode != 0)
{
throw new NotSupportedException($"Can't use 'tar -xzf' extract archive file: {archiveFile}. return code: {exitCode}.");
}
}
#endif
// repository archive from github always contains a nested folder
var subDirectories = new DirectoryInfo(stagingDirectory).GetDirectories();
if (subDirectories.Length != 1)
{
throw new InvalidOperationException($"'{archiveFile}' contains '{subDirectories.Length}' directories");
}
else
{
executionContext.Debug($"Unwrap '{subDirectories[0].Name}' to '{destDirectory}'");
IOUtil.CopyDirectory(subDirectories[0].FullName, destDirectory, executionContext.CancellationToken);
}
Trace.Verbose("Create watermark file indicate action download succeed.");
string watermarkFile = GetWatermarkFilePath(destDirectory);
File.WriteAllText(watermarkFile, DateTime.UtcNow.ToString());
executionContext.Debug($"Archive '{archiveFile}' has been unzipped into '{destDirectory}'.");
Trace.Info("Finished getting action repository.");
}
finally
{
try
{
//if the temp folder wasn't moved -> wipe it
if (Directory.Exists(tempDirectory))
{
Trace.Verbose("Deleting action temp folder: {0}", tempDirectory);
IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast.
}
}
catch (Exception ex)
{
//it is not critical if we fail to delete the temp folder
Trace.Warning("Failed to delete temp folder '{0}'. Exception: {1}", tempDirectory, ex);
}
}
}
private async Task PullContainerAsync(IExecutionContext executionContext, ContainerInfo container)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(container, nameof(container));
ArgUtil.NotNullOrEmpty(container.ContainerImage, nameof(container.ContainerImage));
Trace.Info($"Container name: {container.ContainerName}");
Trace.Info($"Container image: {container.ContainerImage}");
Trace.Info($"Container registry: {container.ContainerRegistryEndpoint.ToString()}");
Trace.Info($"Container options: {container.ContainerCreateOptions}");
Trace.Info($"Skip container image pull: {container.SkipContainerImagePull}");
// Login to private docker registry
string registryServer = string.Empty;
if (container.ContainerRegistryEndpoint != Guid.Empty)
{
var registryEndpoint = executionContext.Endpoints.FirstOrDefault(x => x.Type == "dockerregistry" && x.Id == container.ContainerRegistryEndpoint);
ArgUtil.NotNull(registryEndpoint, nameof(registryEndpoint));
string username = string.Empty;
string password = string.Empty;
string registryType = string.Empty;
registryEndpoint.Data?.TryGetValue("registrytype", out registryType);
if (string.Equals(registryType, "ACR", StringComparison.OrdinalIgnoreCase))
{
string loginServer = string.Empty;
registryEndpoint.Authorization?.Parameters?.TryGetValue("loginServer", out loginServer);
if (loginServer != null)
{
loginServer = loginServer.ToLower();
}
registryEndpoint.Authorization?.Parameters?.TryGetValue("serviceprincipalid", out username);
registryEndpoint.Authorization?.Parameters?.TryGetValue("serviceprincipalkey", out password);
registryServer = $"https://{loginServer}";
}
else
{
registryEndpoint.Authorization?.Parameters?.TryGetValue("registry", out registryServer);
registryEndpoint.Authorization?.Parameters?.TryGetValue("username", out username);
registryEndpoint.Authorization?.Parameters?.TryGetValue("password", out password);
}
ArgUtil.NotNullOrEmpty(registryServer, nameof(registryServer));
ArgUtil.NotNullOrEmpty(username, nameof(username));
ArgUtil.NotNullOrEmpty(password, nameof(password));
int loginExitCode = await _dockerManger.DockerLogin(executionContext, registryServer, username, password);
if (loginExitCode != 0)
{
throw new InvalidOperationException($"Docker login fail with exit code {loginExitCode}");
}
}
try
{
if (!container.SkipContainerImagePull)
{
if (!string.IsNullOrEmpty(registryServer) &&
registryServer.IndexOf("index.docker.io", StringComparison.OrdinalIgnoreCase) < 0)
{
var registryServerUri = new Uri(registryServer);
if (!container.ContainerImage.StartsWith(registryServerUri.Authority, StringComparison.OrdinalIgnoreCase))
{
container.ContainerImage = $"{registryServerUri.Authority}/{container.ContainerImage}";
}
}
// Pull down docker image with retry up to 3 times
int retryCount = 0;
int pullExitCode = 0;
while (retryCount < 3)
{
pullExitCode = await _dockerManger.DockerPull(executionContext, container.ContainerImage);
if (pullExitCode == 0)
{
break;
}
else
{
retryCount++;
if (retryCount < 3)
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
executionContext.Warning($"Docker pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
}
if (retryCount == 3 && pullExitCode != 0)
{
throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}");
}
}
if (PlatformUtil.RunningOnMacOS)
{
container.ImageOS = PlatformUtil.OS.Linux;
}
// if running on Windows, and attempting to run linux container, require container to have node
else if (PlatformUtil.RunningOnWindows)
{
string containerOS = await _dockerManger.DockerInspect(context : executionContext,
dockerObject : container.ContainerImage,
options : $"--format=\"{{{{.Os}}}}\"");
if (string.Equals("linux", containerOS, StringComparison.OrdinalIgnoreCase))
{
container.ImageOS = PlatformUtil.OS.Linux;
}
}
}
finally
{
// Logout for private registry
if (!string.IsNullOrEmpty(registryServer))
{
int logoutExitCode = await _dockerManger.DockerLogout(executionContext, registryServer);
if (logoutExitCode != 0)
{
executionContext.Error($"Docker logout fail with exit code {logoutExitCode}");
}
}
}
}
private async Task StartContainerAsync(IExecutionContext executionContext, ContainerInfo container)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(container, nameof(container));
ArgUtil.NotNullOrEmpty(container.ContainerImage, nameof(container.ContainerImage));
Trace.Info($"Container name: {container.ContainerName}");
Trace.Info($"Container image: {container.ContainerImage}");
Trace.Info($"Container registry: {container.ContainerRegistryEndpoint.ToString()}");
Trace.Info($"Container options: {container.ContainerCreateOptions}");
Trace.Info($"Skip container image pull: {container.SkipContainerImagePull}");
foreach (var port in container.UserPortMappings)
{
Trace.Info($"User provided port: {port.Value}");
}
foreach (var volume in container.UserMountVolumes)
{
Trace.Info($"User provided volume: {volume.Value}");
}
// Login to private docker registry
string registryServer = string.Empty;
if (container.ContainerRegistryEndpoint != Guid.Empty)
{
var registryEndpoint = executionContext.Endpoints.FirstOrDefault(x => x.Type == "dockerregistry" && x.Id == container.ContainerRegistryEndpoint);
ArgUtil.NotNull(registryEndpoint, nameof(registryEndpoint));
string username = string.Empty;
string password = string.Empty;
string registryType = string.Empty;
registryEndpoint.Data?.TryGetValue("registrytype", out registryType);
if (string.Equals(registryType, "ACR", StringComparison.OrdinalIgnoreCase))
{
string loginServer = string.Empty;
registryEndpoint.Authorization?.Parameters?.TryGetValue("loginServer", out loginServer);
registryEndpoint.Authorization?.Parameters?.TryGetValue("serviceprincipalid", out username);
registryEndpoint.Authorization?.Parameters?.TryGetValue("serviceprincipalkey", out password);
registryServer = $"https://{loginServer}";
}
else
{
registryEndpoint.Authorization?.Parameters?.TryGetValue("registry", out registryServer);
registryEndpoint.Authorization?.Parameters?.TryGetValue("username", out username);
registryEndpoint.Authorization?.Parameters?.TryGetValue("password", out password);
}
ArgUtil.NotNullOrEmpty(registryServer, nameof(registryServer));
ArgUtil.NotNullOrEmpty(username, nameof(username));
ArgUtil.NotNullOrEmpty(password, nameof(password));
int loginExitCode = await _dockerManger.DockerLogin(executionContext, registryServer, username, password);
if (loginExitCode != 0)
{
throw new InvalidOperationException($"Docker login fail with exit code {loginExitCode}");
}
}
try
{
if (!container.SkipContainerImagePull)
{
if (!string.IsNullOrEmpty(registryServer) &&
registryServer.IndexOf("index.docker.io", StringComparison.OrdinalIgnoreCase) < 0)
{
var registryServerUri = new Uri(registryServer);
if (!container.ContainerImage.StartsWith(registryServerUri.Authority, StringComparison.OrdinalIgnoreCase))
{
container.ContainerImage = $"{registryServerUri.Authority}/{container.ContainerImage}";
}
}
// Pull down docker image with retry up to 3 times
int retryCount = 0;
int pullExitCode = 0;
while (retryCount < 3)
{
pullExitCode = await _dockerManger.DockerPull(executionContext, container.ContainerImage);
if (pullExitCode == 0)
{
break;
}
else
{
retryCount++;
if (retryCount < 3)
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
executionContext.Warning($"Docker pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
}
if (retryCount == 3 && pullExitCode != 0)
{
throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}");
}
}
// Mount folder into container
#if OS_WINDOWS
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Work), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Work))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools))));
#else
string defaultWorkingDirectory = executionContext.Variables.Get(Constants.Variables.System.DefaultWorkingDirectory);
if (string.IsNullOrEmpty(defaultWorkingDirectory))
{
throw new NotSupportedException(StringUtil.Loc("ContainerJobRequireSystemDefaultWorkDir"));
}
string workingDirectory = Path.GetDirectoryName(defaultWorkingDirectory.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar));
container.MountVolumes.Add(new MountVolume(container.TranslateToHostPath(workingDirectory), workingDirectory));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Temp), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Temp))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tools), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tools))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Tasks), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Tasks))));
container.MountVolumes.Add(new MountVolume(HostContext.GetDirectory(WellKnownDirectory.Externals), container.TranslateToContainerPath(HostContext.GetDirectory(WellKnownDirectory.Externals)), true));
// Ensure .taskkey file exist so we can mount it.
string taskKeyFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), ".taskkey");
if (!File.Exists(taskKeyFile))
{
File.WriteAllText(taskKeyFile, string.Empty);
}
container.MountVolumes.Add(new MountVolume(taskKeyFile, container.TranslateToContainerPath(taskKeyFile)));
#endif
if (container.IsJobContainer)
{
// See if this container brings its own Node.js
container.ContainerBringNodePath = await _dockerManger.DockerInspect(context : executionContext,
dockerObject : container.ContainerImage,
options : $"--format=\"{{{{index .Config.Labels \\\"{_nodeJsPathLabel}\\\"}}}}\"");
string node;
if (!string.IsNullOrEmpty(container.ContainerBringNodePath))
{
node = container.ContainerBringNodePath;
}
else
{
node = container.TranslateToContainerPath(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node", "bin", $"node{IOUtil.ExeExtension}"));
}
string sleepCommand = $"\"{node}\" -e \"setInterval(function(){{}}, 24 * 60 * 60 * 1000);\"";
container.ContainerCommand = sleepCommand;
}
container.ContainerId = await _dockerManger.DockerCreate(executionContext, container);
ArgUtil.NotNullOrEmpty(container.ContainerId, nameof(container.ContainerId));
if (container.IsJobContainer)
{
executionContext.Variables.Set(Constants.Variables.Agent.ContainerId, container.ContainerId);
}
// Start container
int startExitCode = await _dockerManger.DockerStart(executionContext, container.ContainerId);
if (startExitCode != 0)
{
throw new InvalidOperationException($"Docker start fail with exit code {startExitCode}");
}
}
finally
{
// Logout for private registry
if (!string.IsNullOrEmpty(registryServer))
{
int logoutExitCode = await _dockerManger.DockerLogout(executionContext, registryServer);
if (logoutExitCode != 0)
{
executionContext.Error($"Docker logout fail with exit code {logoutExitCode}");
}
}
}
try
{
// Make sure container is up and running
var psOutputs = await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --filter status=running --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\"");
if (psOutputs.FirstOrDefault(x => !string.IsNullOrEmpty(x))?.StartsWith(container.ContainerId) != true)
{
// container is not up and running, pull docker log for this container.
await _dockerManger.DockerPS(executionContext, $"--all --filter id={container.ContainerId} --no-trunc --format \"{{{{.ID}}}} {{{{.Status}}}}\"");
int logsExitCode = await _dockerManger.DockerLogs(executionContext, container.ContainerId);
if (logsExitCode != 0)
{
executionContext.Warning($"Docker logs fail with exit code {logsExitCode}");
}
executionContext.Warning($"Docker container {container.ContainerId} is not in running state.");
}
}
catch (Exception ex)
{
// pull container log is best effort.
Trace.Error("Catch exception when check container log and container status.");
Trace.Error(ex);
}
// Get port mappings of running container
if (executionContext.Container == null && !container.IsJobContainer)
{
container.AddPortMappings(await _dockerManger.DockerPort(executionContext, container.ContainerId));
foreach (var port in container.PortMappings)
{
executionContext.Variables.Set(
$"{Constants.Variables.Agent.ServicePortPrefix}.{container.ContainerNetworkAlias}.ports.{port.ContainerPort}",
$"{port.HostPort}");
}
}
#if !OS_WINDOWS
if (container.IsJobContainer)
{
// Ensure bash exist in the image
int execWhichBashExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"sh -c \"command -v bash\"");
if (execWhichBashExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execWhichBashExitCode}");
}
// Get current username
container.CurrentUserName = (await ExecuteCommandAsync(executionContext, "whoami", string.Empty)).FirstOrDefault();
ArgUtil.NotNullOrEmpty(container.CurrentUserName, nameof(container.CurrentUserName));
// Get current userId
container.CurrentUserId = (await ExecuteCommandAsync(executionContext, "id", $"-u {container.CurrentUserName}")).FirstOrDefault();
ArgUtil.NotNullOrEmpty(container.CurrentUserId, nameof(container.CurrentUserId));
executionContext.Output(StringUtil.Loc("CreateUserWithSameUIDInsideContainer", container.CurrentUserId));
// Create an user with same uid as the agent run as user inside the container.
// All command execute in docker will run as Root by default,
// this will cause the agent on the host machine doesn't have permission to any new file/folder created inside the container.
// So, we create a user account with same UID inside the container and let all docker exec command run as that user.
string containerUserName = string.Empty;
// We need to find out whether there is a user with same UID inside the container
List <string> userNames = new List <string>();
int execGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"grep {container.CurrentUserId} /etc/passwd | cut -f1 -d:\"", userNames);
if (execGrepExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGrepExitCode}");
}
if (userNames.Count > 0)
{
// check all potential username that might match the UID.
foreach (string username in userNames)
{
int execIdExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"id -u {username}");
if (execIdExitCode == 0)
{
containerUserName = username;
break;
}
}
}
// Create a new user with same UID
if (string.IsNullOrEmpty(containerUserName))
{
containerUserName = $"{container.CurrentUserName}_azpcontainer";
int execUseraddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"useradd -m -u {container.CurrentUserId} {containerUserName}");
if (execUseraddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execUseraddExitCode}");
}
}
executionContext.Output(StringUtil.Loc("GrantContainerUserSUDOPrivilege", containerUserName));
// Create a new group for giving sudo permission
int execGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd azure_pipelines_sudo");
if (execGroupaddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGroupaddExitCode}");
}
// Add the new created user to the new created sudo group.
int execUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G azure_pipelines_sudo {containerUserName}");
if (execUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execUsermodExitCode}");
}
// Allow the new sudo group run any sudo command without providing password.
int execEchoExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"su -c \"echo '%azure_pipelines_sudo ALL=(ALL:ALL) NOPASSWD:ALL' >> /etc/sudoers\"");
if (execUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execEchoExitCode}");
}
bool setupDockerGroup = executionContext.Variables.GetBoolean("VSTS_SETUP_DOCKERGROUP") ?? StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("VSTS_SETUP_DOCKERGROUP"), true);
if (setupDockerGroup)
{
executionContext.Output(StringUtil.Loc("AllowContainerUserRunDocker", containerUserName));
// Get docker.sock group id on Host
string dockerSockGroupId = (await ExecuteCommandAsync(executionContext, "stat", $"-c %g /var/run/docker.sock")).FirstOrDefault();
// We need to find out whether there is a group with same GID inside the container
string existingGroupName = null;
List <string> groupsOutput = new List <string>();
int execGroupGrepExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"bash -c \"cat /etc/group\"", groupsOutput);
if (execGroupGrepExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGroupGrepExitCode}");
}
if (groupsOutput.Count > 0)
{
// check all potential groups that might match the GID.
foreach (string groupOutput in groupsOutput)
{
if (!string.IsNullOrEmpty(groupOutput))
{
var groupSegments = groupOutput.Split(':');
if (groupSegments.Length != 4)
{
Trace.Warning($"Unexpected output from /etc/group: '{groupOutput}'");
}
else
{
// the output of /etc/group should looks like `group:x:gid:`
var groupName = groupSegments[0];
var groupId = groupSegments[2];
if (string.Equals(dockerSockGroupId, groupId))
{
existingGroupName = groupName;
break;
}
}
}
}
}
if (string.IsNullOrEmpty(existingGroupName))
{
// create a new group with same gid
existingGroupName = "azure_pipelines_docker";
int execDockerGroupaddExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"groupadd -g {dockerSockGroupId} azure_pipelines_docker");
if (execDockerGroupaddExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execDockerGroupaddExitCode}");
}
}
// Add the new created user to the docker socket group.
int execGroupUsermodExitCode = await _dockerManger.DockerExec(executionContext, container.ContainerId, string.Empty, $"usermod -a -G {existingGroupName} {containerUserName}");
if (execGroupUsermodExitCode != 0)
{
throw new InvalidOperationException($"Docker exec fail with exit code {execGroupUsermodExitCode}");
}
}
}
#endif
}
private async Task <VssCredentials> GetNewOAuthAuthorizationSetting(CancellationToken token)
{
Trace.Info("Start checking oauth authorization url update.");
while (true)
{
var backoff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromMinutes(30), TimeSpan.FromMinutes(45));
await HostContext.Delay(backoff, token);
try
{
var migratedAuthorizationUrl = await _runnerServer.GetRunnerAuthUrlAsync(_settings.PoolId, _settings.AgentId);
if (!string.IsNullOrEmpty(migratedAuthorizationUrl))
{
var credData = _configStore.GetCredentials();
var clientId = credData.Data.GetValueOrDefault("clientId", null);
var currentAuthorizationUrl = credData.Data.GetValueOrDefault("authorizationUrl", null);
Trace.Info($"Current authorization url: {currentAuthorizationUrl}, new authorization url: {migratedAuthorizationUrl}");
if (string.Equals(currentAuthorizationUrl, migratedAuthorizationUrl, StringComparison.OrdinalIgnoreCase))
{
// We don't need to update credentials.
Trace.Info("No needs to update authorization url");
await Task.Delay(TimeSpan.FromMilliseconds(-1), token);
}
var keyManager = HostContext.GetService <IRSAKeyManager>();
var signingCredentials = VssSigningCredentials.Create(() => keyManager.GetKey());
var migratedClientCredential = new VssOAuthJwtBearerClientCredential(clientId, migratedAuthorizationUrl, signingCredentials);
var migratedRunnerCredential = new VssOAuthCredential(new Uri(migratedAuthorizationUrl, UriKind.Absolute), VssOAuthGrant.ClientCredentials, migratedClientCredential);
Trace.Info("Try connect service with Token Service OAuth endpoint.");
var runnerServer = HostContext.CreateService <IRunnerServer>();
await runnerServer.ConnectAsync(new Uri(_settings.ServerUrl), migratedRunnerCredential);
await runnerServer.GetAgentPoolsAsync();
Trace.Info($"Successfully connected service with new authorization url.");
var migratedCredData = new CredentialData
{
Scheme = Constants.Configuration.OAuth,
Data =
{
{ "clientId", clientId },
{ "authorizationUrl", migratedAuthorizationUrl },
{ "oauthEndpointUrl", migratedAuthorizationUrl },
},
};
_configStore.SaveMigratedCredential(migratedCredData);
return(migratedRunnerCredential);
}
else
{
Trace.Verbose("No authorization url updates");
}
}
catch (Exception ex)
{
Trace.Error("Fail to get/test new authorization url.");
Trace.Error(ex);
try
{
await _runnerServer.ReportRunnerAuthUrlErrorAsync(_settings.PoolId, _settings.AgentId, ex.ToString());
}
catch (Exception e)
{
// best effort
Trace.Error("Fail to report the migration error");
Trace.Error(e);
}
}
}
}
public async Task <TaskAgentMessage> GetNextMessageAsync(CancellationToken token)
{
Trace.Entering();
ArgUtil.NotNull(_session, nameof(_session));
ArgUtil.NotNull(_settings, nameof(_settings));
bool encounteringError = false;
int continuousError = 0;
string errorMessage = string.Empty;
Stopwatch heartbeat = new Stopwatch();
heartbeat.Restart();
while (true)
{
token.ThrowIfCancellationRequested();
TaskAgentMessage message = null;
try
{
message = await _runnerServer.GetAgentMessageAsync(_settings.PoolId,
_session.SessionId,
_lastMessageId,
token);
// Decrypt the message body if the session is using encryption
message = DecryptMessage(message);
if (message != null)
{
_lastMessageId = message.MessageId;
}
if (encounteringError) //print the message once only if there was an error
{
_term.WriteLine($"{DateTime.UtcNow:u}: Runner reconnected.");
encounteringError = false;
continuousError = 0;
}
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
Trace.Info("Get next message has been cancelled.");
throw;
}
catch (TaskAgentAccessTokenExpiredException)
{
Trace.Info("Runner OAuth token has been revoked. Unable to pull message.");
throw;
}
catch (Exception ex)
{
Trace.Error("Catch exception during get next message.");
Trace.Error(ex);
// don't retry if SkipSessionRecover = true, DT service will delete agent session to stop agent from taking more jobs.
if (ex is TaskAgentSessionExpiredException && !_settings.SkipSessionRecover && await CreateSessionAsync(token))
{
Trace.Info($"{nameof(TaskAgentSessionExpiredException)} received, recovered by recreate session.");
}
else if (!IsGetNextMessageExceptionRetriable(ex))
{
throw;
}
else
{
continuousError++;
//retry after a random backoff to avoid service throttling
//in case of there is a service error happened and all agents get kicked off of the long poll and all agent try to reconnect back at the same time.
if (continuousError <= 5)
{
// random backoff [15, 30]
_getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30), _getNextMessageRetryInterval);
}
else
{
// more aggressive backoff [30, 60]
_getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(60), _getNextMessageRetryInterval);
}
if (!encounteringError)
{
//print error only on the first consecutive error
_term.WriteError($"{DateTime.UtcNow:u}: Runner connect error: {ex.Message}. Retrying until reconnected.");
encounteringError = true;
}
// re-create VssConnection before next retry
await _runnerServer.RefreshConnectionAsync(RunnerConnectionType.MessageQueue, TimeSpan.FromSeconds(60));
Trace.Info("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds);
await HostContext.Delay(_getNextMessageRetryInterval, token);
}
}
if (message == null)
{
if (heartbeat.Elapsed > TimeSpan.FromMinutes(30))
{
Trace.Info($"No message retrieved from session '{_session.SessionId}' within last 30 minutes.");
heartbeat.Restart();
}
else
{
Trace.Verbose($"No message retrieved from session '{_session.SessionId}'.");
}
continue;
}
Trace.Info($"Message '{message.MessageId}' received from session '{_session.SessionId}'.");
return(message);
}
}
public async Task <Boolean> CreateSessionAsync(CancellationToken token)
{
Trace.Entering();
// Settings
var configManager = HostContext.GetService <IConfigurationManager>();
_settings = configManager.LoadSettings();
var serverUrl = _settings.ServerUrl;
Trace.Info(_settings);
// Create connection.
Trace.Info("Loading Credentials");
_useMigratedCredentials = !StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_SPSAUTHURL"));
VssCredentials creds = _credMgr.LoadCredentials(_useMigratedCredentials);
var agent = new TaskAgentReference
{
Id = _settings.AgentId,
Name = _settings.AgentName,
Version = BuildConstants.RunnerPackage.Version,
OSDescription = RuntimeInformation.OSDescription,
};
string sessionName = $"{Environment.MachineName ?? "RUNNER"}";
var taskAgentSession = new TaskAgentSession(sessionName, agent);
string errorMessage = string.Empty;
bool encounteringError = false;
var originalCreds = _configStore.GetCredentials();
var migratedCreds = _configStore.GetMigratedCredentials();
if (migratedCreds == null)
{
_useMigratedCredentials = false;
if (originalCreds.Scheme == Constants.Configuration.OAuth)
{
_needToCheckAuthorizationUrlUpdate = true;
}
}
while (true)
{
token.ThrowIfCancellationRequested();
Trace.Info($"Attempt to create session.");
try
{
Trace.Info("Connecting to the Runner Server...");
await _runnerServer.ConnectAsync(new Uri(serverUrl), creds);
Trace.Info("VssConnection created");
_term.WriteLine();
_term.WriteSuccessMessage("Connected to GitHub");
_term.WriteLine();
_session = await _runnerServer.CreateAgentSessionAsync(
_settings.PoolId,
taskAgentSession,
token);
Trace.Info($"Session created.");
if (encounteringError)
{
_term.WriteLine($"{DateTime.UtcNow:u}: Runner reconnected.");
_sessionCreationExceptionTracker.Clear();
encounteringError = false;
}
if (_needToCheckAuthorizationUrlUpdate)
{
// start background task try to get new authorization url
_authorizationUrlMigrationBackgroundTask = GetNewOAuthAuthorizationSetting(token);
}
return(true);
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
Trace.Info("Session creation has been cancelled.");
throw;
}
catch (TaskAgentAccessTokenExpiredException)
{
Trace.Info("Runner OAuth token has been revoked. Session creation failed.");
throw;
}
catch (Exception ex)
{
Trace.Error("Catch exception during create session.");
Trace.Error(ex);
if (!IsSessionCreationExceptionRetriable(ex))
{
if (_useMigratedCredentials)
{
// migrated credentials might cause lose permission during permission check,
// we will force to use original credential and try again
_useMigratedCredentials = false;
var reattemptBackoff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromHours(24), TimeSpan.FromHours(36));
_authorizationUrlRollbackReattemptDelayBackgroundTask = HostContext.Delay(reattemptBackoff, token); // retry migrated creds in 24-36 hours.
creds = _credMgr.LoadCredentials(false);
Trace.Error("Fallback to original credentials and try again.");
}
else
{
_term.WriteError($"Failed to create session. {ex.Message}");
return(false);
}
}
if (!encounteringError) //print the message only on the first error
{
_term.WriteError($"{DateTime.UtcNow:u}: Runner connect error: {ex.Message}. Retrying until reconnected.");
encounteringError = true;
}
Trace.Info("Sleeping for {0} seconds before retrying.", _sessionCreationRetryInterval.TotalSeconds);
await HostContext.Delay(_sessionCreationRetryInterval, token);
}
}
}
// git fetch --tags --prune --progress --no-recurse-submodules [--depth=15] origin [+refs/pull/*:refs/remote/pull/*]
public async Task <int> GitFetch(AgentTaskPluginExecutionContext context, string repositoryPath, string remoteName, int fetchDepth, List <string> refSpec, string additionalCommandLine, CancellationToken cancellationToken)
{
context.Debug($"Fetch git repository at: {repositoryPath} remote: {remoteName}.");
if (refSpec != null && refSpec.Count > 0)
{
refSpec = refSpec.Where(r => !string.IsNullOrEmpty(r)).ToList();
}
// Git 2.20 changed its fetch behavior, rejecting tag updates if the --force flag is not provided
// See https://git-scm.com/docs/git-fetch for more details
string forceTag = string.Empty;
if (gitVersion >= new Version(2, 20))
{
forceTag = "--force";
}
// default options for git fetch.
string options = StringUtil.Format($"{forceTag} --tags --prune --progress --no-recurse-submodules {remoteName} {string.Join(" ", refSpec)}");
// If shallow fetch add --depth arg
// If the local repository is shallowed but there is no fetch depth provide for this build,
// add --unshallow to convert the shallow repository to a complete repository
if (fetchDepth > 0)
{
options = StringUtil.Format($"{forceTag} --tags --prune --progress --no-recurse-submodules --depth={fetchDepth} {remoteName} {string.Join(" ", refSpec)}");
}
else
{
if (File.Exists(Path.Combine(repositoryPath, ".git", "shallow")))
{
options = StringUtil.Format($"{forceTag} --tags --prune --progress --no-recurse-submodules --unshallow {remoteName} {string.Join(" ", refSpec)}");
}
}
int retryCount = 0;
int fetchExitCode = 0;
while (retryCount < 3)
{
Stopwatch watch = new Stopwatch();
watch.Start();
fetchExitCode = await ExecuteGitCommandAsync(context, repositoryPath, "fetch", options, additionalCommandLine, cancellationToken);
watch.Stop();
// Publish some fetch statistics
context.PublishTelemetry(area: "AzurePipelinesAgent", feature: "GitFetch", properties: new Dictionary <string, string>
{
{ "ElapsedTimeMilliseconds", $"{watch.ElapsedMilliseconds}" },
{ "RefSpec", string.Join(" ", refSpec) },
{ "RemoteName", remoteName },
{ "FetchDepth", $"{fetchDepth}" },
{ "ExitCode", $"{fetchExitCode}" },
{ "Options", options }
});
if (fetchExitCode == 0)
{
break;
}
else
{
if (++retryCount < 3)
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
context.Warning($"Git fetch failed with exit code {fetchExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
}
return(fetchExitCode);
}
protected override async Task <HttpResponseMessage> SendAsync(
HttpRequestMessage request,
CancellationToken cancellationToken)
{
ArgUtil.NotNull(request, nameof(request));
if (PlatformUtil.RunningOnMacOS || PlatformUtil.RunningOnLinux)
{
request.Version = HttpVersion.Version11;
}
// Potential improvements:
// 1. Calculate a max time across attempts, to avoid retries (this class) on top of retries (VssHttpRetryMessageHandler)
// causing more time to pass than expected in degenerative cases.
// 2. Increase the per-attempt timeout on each attempt. Instead of 5 minutes on each attempt, start low and build to 10-20 minutes.
HttpResponseMessage response = null;
// We can safely retry on timeout if the request isn't one that changes state.
Boolean canRetry = (request.Method == HttpMethod.Get || request.Method == HttpMethod.Head || request.Method == HttpMethod.Options);
if (canRetry)
{
Int32 attempt = 1;
TimeoutException exception = null;
Int32 maxAttempts = _retryOptions.MaxRetries + 1;
while (attempt <= maxAttempts)
{
// Reset the exception so we don't have a lingering variable
exception = null;
Stopwatch watch = Stopwatch.StartNew();
try
{
response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false);
break;
}
catch (TimeoutException ex)
{
exception = ex;
}
TimeSpan backoff;
if (attempt < maxAttempts)
{
backoff = BackoffTimerHelper.GetExponentialBackoff(
attempt,
_retryOptions.MinBackoff,
_retryOptions.MaxBackoff,
_retryOptions.BackoffCoefficient);
}
else
{
break;
}
string message = StringUtil.Loc("RMContainerItemRequestTimedOut", (int)watch.Elapsed.TotalSeconds, backoff.TotalSeconds, request.Method, request.RequestUri);
_logger.Warning(message);
attempt++;
await Task.Delay(backoff, cancellationToken).ConfigureAwait(false);
}
if (exception != null)
{
throw exception;
}
}
else
{
// No retries. Just pipe the request through to the other handlers.
response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false);
}
return(response);
}
public async Task RenewJobRequestAsync(int poolId, long requestId, Guid lockToken, TaskCompletionSource <int> firstJobRequestRenewed, CancellationToken token)
{
var agentServer = HostContext.GetService <IAgentServer>();
TaskAgentJobRequest request = null;
int firstRenewRetryLimit = 5;
int encounteringError = 0;
// renew lock during job running.
// stop renew only if cancellation token for lock renew task been signal or exception still happen after retry.
while (!token.IsCancellationRequested)
{
try
{
request = await agentServer.RenewAgentRequestAsync(poolId, requestId, lockToken, token);
Trace.Info($"Successfully renew job request {requestId}, job is valid till {request.LockedUntil.Value}");
if (!firstJobRequestRenewed.Task.IsCompleted)
{
// fire first renew succeed event.
firstJobRequestRenewed.TrySetResult(0);
}
if (encounteringError > 0)
{
encounteringError = 0;
agentServer.SetConnectionTimeout(AgentConnectionType.JobRequest, TimeSpan.FromSeconds(60));
HostContext.WritePerfCounter("JobRenewRecovered");
}
// renew again after 60 sec delay
await HostContext.Delay(TimeSpan.FromSeconds(60), token);
}
catch (TaskAgentJobNotFoundException)
{
// no need for retry. the job is not valid anymore.
Trace.Info($"TaskAgentJobNotFoundException received when renew job request {requestId}, job is no longer valid, stop renew job request.");
return;
}
catch (TaskAgentJobTokenExpiredException)
{
// no need for retry. the job is not valid anymore.
Trace.Info($"TaskAgentJobTokenExpiredException received renew job request {requestId}, job is no longer valid, stop renew job request.");
return;
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
// OperationCanceledException may caused by http timeout or _lockRenewalTokenSource.Cance();
// Stop renew only on cancellation token fired.
Trace.Info($"job renew has been canceled, stop renew job request {requestId}.");
return;
}
catch (Exception ex)
{
Trace.Error($"Catch exception during renew agent jobrequest {requestId}.");
Trace.Error(ex);
encounteringError++;
// retry
TimeSpan remainingTime = TimeSpan.Zero;
if (!firstJobRequestRenewed.Task.IsCompleted)
{
// retry 5 times every 10 sec for the first renew
if (firstRenewRetryLimit-- > 0)
{
remainingTime = TimeSpan.FromSeconds(10);
}
}
else
{
// retry till reach lockeduntil + 5 mins extra buffer.
remainingTime = request.LockedUntil.Value + TimeSpan.FromMinutes(5) - DateTime.UtcNow;
}
if (remainingTime > TimeSpan.Zero)
{
TimeSpan delayTime;
if (!firstJobRequestRenewed.Task.IsCompleted)
{
Trace.Info($"Retrying lock renewal for jobrequest {requestId}. The first job renew request has failed.");
delayTime = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
}
else
{
Trace.Info($"Retrying lock renewal for jobrequest {requestId}. Job is valid until {request.LockedUntil.Value}.");
if (encounteringError > 5)
{
delayTime = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30));
}
else
{
delayTime = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
}
}
// Re-establish connection to server in order to avoid affinity with server.
// Reduce connection timeout to 30 seconds (from 60s)
HostContext.WritePerfCounter("ResetJobRenewConnection");
await agentServer.RefreshConnectionAsync(AgentConnectionType.JobRequest, TimeSpan.FromSeconds(30));
try
{
// back-off before next retry.
await HostContext.Delay(delayTime, token);
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
Trace.Info($"job renew has been canceled, stop renew job request {requestId}.");
}
}
else
{
Trace.Info($"Lock renewal has run out of retry, stop renew lock for jobrequest {requestId}.");
HostContext.WritePerfCounter("JobRenewReachLimit");
return;
}
}
}
}
private async Task <DownloadResult> DownloadAsync(RunnerActionPluginExecutionContext context, int downloaderId, CancellationToken token)
{
List <DownloadInfo> failedFiles = new List <DownloadInfo>();
Stopwatch downloadTimer = new Stopwatch();
while (_fileDownloadQueue.TryDequeue(out DownloadInfo fileToDownload))
{
token.ThrowIfCancellationRequested();
try
{
int retryCount = 0;
bool downloadFailed = false;
while (true)
{
try
{
context.Debug($"Start downloading file: '{fileToDownload.ItemPath}' (Downloader {downloaderId})");
downloadTimer.Restart();
using (FileStream fs = new FileStream(fileToDownload.LocalPath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true))
using (var downloadStream = await _fileContainerHttpClient.DownloadFileAsync(_containerId, fileToDownload.ItemPath, token, _projectId))
{
await downloadStream.CopyToAsync(fs, _defaultCopyBufferSize, token);
await fs.FlushAsync(token);
downloadTimer.Stop();
context.Debug($"File: '{fileToDownload.LocalPath}' took {downloadTimer.ElapsedMilliseconds} milliseconds to finish download (Downloader {downloaderId})");
break;
}
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
context.Debug($"Download has been cancelled while downloading {fileToDownload.ItemPath}. (Downloader {downloaderId})");
throw;
}
catch (Exception ex)
{
retryCount++;
context.Warning($"Fail to download '{fileToDownload.ItemPath}', error: {ex.Message} (Downloader {downloaderId})");
context.Debug(ex.ToString());
}
if (retryCount < 3)
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30));
context.Warning($"Back off {backOff.TotalSeconds} seconds before retry. (Downloader {downloaderId})");
await Task.Delay(backOff);
}
else
{
// upload still failed after 3 tries.
downloadFailed = true;
break;
}
}
if (downloadFailed)
{
// tracking file that failed to download.
failedFiles.Add(fileToDownload);
}
Interlocked.Increment(ref _downloadFilesProcessed);
}
catch (Exception ex)
{
// We should never
context.Error($"Error '{ex.Message}' when downloading file '{fileToDownload}'. (Downloader {downloaderId})");
throw;
}
}
return(new DownloadResult(failedFiles));
}
