private async Task <string> GetAzureSubscriptionIdAsync()
{
// We will use the Azure Instance Metadata Service in order to fetch metadata ( in this case Subscription Id used to provision the VM) if the VM is an Azure VM
// More on Instance Metadata Service can be found here: https://docs.microsoft.com/en-us/azure/virtual-machines/windows/instance-metadata-service
string azureSubscriptionId = string.Empty;
const string imdsUri = "http://169.254.169.254/metadata/instance/compute/subscriptionId?api-version=2017-08-01&format=text";
using (var handler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(handler))
{
httpClient.DefaultRequestHeaders.Add("Metadata", "True");
httpClient.Timeout = TimeSpan.FromSeconds(5);
try
{
azureSubscriptionId = await httpClient.GetStringAsync(imdsUri);
if (!Guid.TryParse(azureSubscriptionId, out Guid result))
{
azureSubscriptionId = string.Empty;
}
}
catch (Exception ex)
{
// An exception will be thrown if the Agent Machine is a non-Azure VM.
azureSubscriptionId = string.Empty;
Trace.Info($"GetAzureSubscriptionId ex: {ex.Message}");
}
}
return(azureSubscriptionId);
}
private async Task <GitHubAuthResult> GetTenantCredential(string githubUrl, string githubToken)
{
var gitHubUrl = new UriBuilder(githubUrl);
var githubApiUrl = $"https://api.{gitHubUrl.Host}/repos/{gitHubUrl.Path.Trim('/')}/actions-runners/registration";
using (var httpClientHandler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(httpClientHandler))
{
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("RemoteAuth", githubToken);
httpClient.DefaultRequestHeaders.UserAgent.Add(HostContext.UserAgent);
httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github.shuri-preview+json"));
var response = await httpClient.PostAsync(githubApiUrl, new StringContent("", null, "application/json"));
if (response.IsSuccessStatusCode)
{
Trace.Info($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
var jsonResponse = await response.Content.ReadAsStringAsync();
return(StringUtil.ConvertFromJson <GitHubAuthResult>(jsonResponse));
}
else
{
_term.WriteError($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
var errorResponse = await response.Content.ReadAsStringAsync();
_term.WriteError(errorResponse);
response.EnsureSuccessStatusCode();
return(null);
}
}
}
private T QueryItem <T>(string accessToken, string url, out string errorMessage)
{
HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, url);
request.Headers.Add("Accept", "application/vnd.GitHubData.V3+json");
request.Headers.Add("Authorization", "Token " + accessToken);
request.Headers.Add("User-Agent", "VSTS-Agent/" + Constants.Agent.Version);
int httpRequestTimeoutSeconds;
if (!int.TryParse(Environment.GetEnvironmentVariable("VSTS_HTTP_TIMEOUT") ?? string.Empty, out httpRequestTimeoutSeconds))
{
httpRequestTimeoutSeconds = 100;
}
using (var httpClientHandler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(httpClientHandler)
{
Timeout = new TimeSpan(0, 0, httpRequestTimeoutSeconds)
})
{
errorMessage = string.Empty;
Task <HttpResponseMessage> sendAsyncTask = httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
HttpResponseMessage response = sendAsyncTask.GetAwaiter().GetResult();
if (!response.IsSuccessStatusCode)
{
errorMessage = response.StatusCode.ToString();
return(default(T));
}
string result = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
return(JsonConvert.DeserializeObject <T>(result));
}
}
private T QueryItem <T>(string accessToken, string url, out string errorMessage)
{
using (var request = new HttpRequestMessage(HttpMethod.Get, url))
{
request.Headers.Add("Accept", "application/vnd.GitHubData.V3+json");
request.Headers.Add("Authorization", "Token " + accessToken);
request.Headers.Add("User-Agent", "VSTS-Agent/" + BuildConstants.AgentPackage.Version);
if (PlatformUtil.RunningOnMacOS || PlatformUtil.RunningOnLinux)
{
request.Version = HttpVersion.Version11;
}
int httpRequestTimeoutSeconds = AgentKnobs.HttpTimeout.GetValue(HostContext).AsInt();
using (var httpClientHandler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(httpClientHandler)
{
Timeout = new TimeSpan(0, 0, httpRequestTimeoutSeconds)
})
{
errorMessage = string.Empty;
Task <HttpResponseMessage> sendAsyncTask = httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
HttpResponseMessage response = sendAsyncTask.GetAwaiter().GetResult();
if (!response.IsSuccessStatusCode)
{
errorMessage = response.StatusCode.ToString();
return(default(T));
}
string result = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
return(JsonConvert.DeserializeObject <T>(result));
}
}
}
public async Task <HttpResponseMessage> GetAsync(string url, string userName, string password, bool acceptUntrustedCertifact)
{
using (HttpClientHandler handler = HostContext.CreateHttpClientHandler())
{
handler.ServerCertificateCustomValidationCallback = (message, certificate, chain, sslPolicyErrors) =>
{
return(acceptUntrustedCertifact);
};
using (HttpClient client = new HttpClient(handler))
{
SetupHttpClient(client, userName, password);
return(await client.GetAsync(url));
}
}
}
private async Task <GitHubAuthResult> GetTenantCredential(string githubUrl, string githubToken, string runnerEvent)
{
var githubApiUrl = "";
var gitHubUrlBuilder = new UriBuilder(githubUrl);
if (IsHostedServer(gitHubUrlBuilder))
{
githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/actions/runner-registration";
}
else
{
githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/actions/runner-registration";
}
using (var httpClientHandler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(httpClientHandler))
{
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("RemoteAuth", githubToken);
httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents);
var bodyObject = new Dictionary <string, string>()
{
{ "url", githubUrl },
{ "runner_event", runnerEvent }
};
var response = await httpClient.PostAsync(githubApiUrl, new StringContent(StringUtil.ConvertToJson(bodyObject), null, "application/json"));
if (response.IsSuccessStatusCode)
{
Trace.Info($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
var jsonResponse = await response.Content.ReadAsStringAsync();
return(StringUtil.ConvertFromJson <GitHubAuthResult>(jsonResponse));
}
else
{
_term.WriteError($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
var errorResponse = await response.Content.ReadAsStringAsync();
_term.WriteError(errorResponse);
response.EnsureSuccessStatusCode();
return(null);
}
}
}
/// <summary>
/// _work
/// \_update
/// \bin
/// \externals
/// \run.sh
/// \run.cmd
/// \package.zip //temp download .zip/.tar.gz
/// </summary>
/// <param name="token"></param>
/// <returns></returns>
private async Task DownloadLatestRunner(CancellationToken token)
{
string latestRunnerDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), Constants.Path.UpdateDirectory);
IOUtil.DeleteDirectory(latestRunnerDirectory, token);
Directory.CreateDirectory(latestRunnerDirectory);
int runnerSuffix = 1;
string archiveFile = null;
bool downloadSucceeded = false;
try
{
// Download the runner, using multiple attempts in order to be resilient against any networking/CDN issues
for (int attempt = 1; attempt <= Constants.RunnerDownloadRetryMaxAttempts; attempt++)
{
// Generate an available package name, and do our best effort to clean up stale local zip files
while (true)
{
if (_targetPackage.Platform.StartsWith("win"))
{
archiveFile = Path.Combine(latestRunnerDirectory, $"runner{runnerSuffix}.zip");
}
else
{
archiveFile = Path.Combine(latestRunnerDirectory, $"runner{runnerSuffix}.tar.gz");
}
try
{
// delete .zip file
if (!string.IsNullOrEmpty(archiveFile) && File.Exists(archiveFile))
{
Trace.Verbose("Deleting latest runner package zip '{0}'", archiveFile);
IOUtil.DeleteFile(archiveFile);
}
break;
}
catch (Exception ex)
{
// couldn't delete the file for whatever reason, so generate another name
Trace.Warning("Failed to delete runner package zip '{0}'. Exception: {1}", archiveFile, ex);
runnerSuffix++;
}
}
// Allow a 15-minute package download timeout, which is good enough to update the runner from a 1 Mbit/s ADSL connection.
if (!int.TryParse(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_DOWNLOAD_TIMEOUT") ?? string.Empty, out int timeoutSeconds))
{
timeoutSeconds = 15 * 60;
}
Trace.Info($"Attempt {attempt}: save latest runner into {archiveFile}.");
using (var downloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
using (var downloadCts = CancellationTokenSource.CreateLinkedTokenSource(downloadTimeout.Token, token))
{
try
{
Trace.Info($"Download runner: begin download");
//open zip stream in async mode
using (HttpClient httpClient = new HttpClient(HostContext.CreateHttpClientHandler()))
using (FileStream fs = new FileStream(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 4096, useAsync: true))
using (Stream result = await httpClient.GetStreamAsync(_targetPackage.DownloadUrl))
{
//81920 is the default used by System.IO.Stream.CopyTo and is under the large object heap threshold (85k).
await result.CopyToAsync(fs, 81920, downloadCts.Token);
await fs.FlushAsync(downloadCts.Token);
}
Trace.Info($"Download runner: finished download");
downloadSucceeded = true;
break;
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
Trace.Info($"Runner download has been canceled.");
throw;
}
catch (Exception ex)
{
if (downloadCts.Token.IsCancellationRequested)
{
Trace.Warning($"Runner download has timed out after {timeoutSeconds} seconds");
}
Trace.Warning($"Failed to get package '{archiveFile}' from '{_targetPackage.DownloadUrl}'. Exception {ex}");
}
}
}
if (!downloadSucceeded)
{
throw new TaskCanceledException($"Runner package '{archiveFile}' failed after {Constants.RunnerDownloadRetryMaxAttempts} download attempts");
}
// If we got this far, we know that we've successfully downloaded the runner package
if (archiveFile.EndsWith(".zip", StringComparison.OrdinalIgnoreCase))
{
ZipFile.ExtractToDirectory(archiveFile, latestRunnerDirectory);
}
else if (archiveFile.EndsWith(".tar.gz", StringComparison.OrdinalIgnoreCase))
{
string tar = WhichUtil.Which("tar", trace: Trace);
if (string.IsNullOrEmpty(tar))
{
throw new NotSupportedException($"tar -xzf");
}
// tar -xzf
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Info(args.Data);
}
});
processInvoker.ErrorDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Error(args.Data);
}
});
int exitCode = await processInvoker.ExecuteAsync(latestRunnerDirectory, tar, $"-xzf \"{archiveFile}\"", null, token);
if (exitCode != 0)
{
throw new NotSupportedException($"Can't use 'tar -xzf' extract archive file: {archiveFile}. return code: {exitCode}.");
}
}
}
else
{
throw new NotSupportedException($"{archiveFile}");
}
Trace.Info($"Finished getting latest runner package at: {latestRunnerDirectory}.");
}
finally
{
try
{
// delete .zip file
if (!string.IsNullOrEmpty(archiveFile) && File.Exists(archiveFile))
{
Trace.Verbose("Deleting latest runner package zip: {0}", archiveFile);
IOUtil.DeleteFile(archiveFile);
}
}
catch (Exception ex)
{
//it is not critical if we fail to delete the .zip file
Trace.Warning("Failed to delete runner package zip '{0}'. Exception: {1}", archiveFile, ex);
}
}
// copy latest runner into runner root folder
// copy bin from _work/_update -> bin.version under root
string binVersionDir = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"{Constants.Path.BinDirectory}.{_targetPackage.Version}");
Directory.CreateDirectory(binVersionDir);
Trace.Info($"Copy {Path.Combine(latestRunnerDirectory, Constants.Path.BinDirectory)} to {binVersionDir}.");
IOUtil.CopyDirectory(Path.Combine(latestRunnerDirectory, Constants.Path.BinDirectory), binVersionDir, token);
// copy externals from _work/_update -> externals.version under root
string externalsVersionDir = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"{Constants.Path.ExternalsDirectory}.{_targetPackage.Version}");
Directory.CreateDirectory(externalsVersionDir);
Trace.Info($"Copy {Path.Combine(latestRunnerDirectory, Constants.Path.ExternalsDirectory)} to {externalsVersionDir}.");
IOUtil.CopyDirectory(Path.Combine(latestRunnerDirectory, Constants.Path.ExternalsDirectory), externalsVersionDir, token);
// copy and replace all .sh/.cmd files
Trace.Info($"Copy any remaining .sh/.cmd files into runner root.");
foreach (FileInfo file in new DirectoryInfo(latestRunnerDirectory).GetFiles() ?? new FileInfo[0])
{
// Copy and replace the file.
file.CopyTo(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), file.Name), true);
}
}
private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, Pipelines.ActionStep repositoryAction)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
var repositoryReference = repositoryAction.Reference as Pipelines.RepositoryPathReference;
ArgUtil.NotNull(repositoryReference, nameof(repositoryReference));
if (string.Equals(repositoryReference.RepositoryType, Pipelines.PipelineConstants.SelfAlias, StringComparison.OrdinalIgnoreCase))
{
Trace.Info($"Repository action is in 'self' repository.");
return;
}
if (!string.Equals(repositoryReference.RepositoryType, Pipelines.RepositoryTypes.GitHub, StringComparison.OrdinalIgnoreCase))
{
throw new NotSupportedException(repositoryReference.RepositoryType);
}
ArgUtil.NotNullOrEmpty(repositoryReference.Name, nameof(repositoryReference.Name));
ArgUtil.NotNullOrEmpty(repositoryReference.Ref, nameof(repositoryReference.Ref));
string destDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), repositoryReference.Name.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), repositoryReference.Ref);
if (File.Exists(destDirectory + ".completed"))
{
executionContext.Debug($"Action '{repositoryReference.Name}@{repositoryReference.Ref}' already downloaded at '{destDirectory}'.");
return;
}
else
{
// make sure we get a clean folder ready to use.
IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken);
Directory.CreateDirectory(destDirectory);
executionContext.Output($"Download action repository '{repositoryReference.Name}@{repositoryReference.Ref}'");
}
#if OS_WINDOWS
string archiveLink = $"https://api.github.com/repos/{repositoryReference.Name}/zipball/{repositoryReference.Ref}";
#else
string archiveLink = $"https://api.github.com/repos/{repositoryReference.Name}/tarball/{repositoryReference.Ref}";
#endif
Trace.Info($"Download archive '{archiveLink}' to '{destDirectory}'.");
//download and extract action in a temp folder and rename it on success
string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), "_temp_" + Guid.NewGuid());
Directory.CreateDirectory(tempDirectory);
#if OS_WINDOWS
string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.zip");
#else
string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.tar.gz");
#endif
Trace.Info($"Save archive '{archiveLink}' into {archiveFile}.");
try
{
int retryCount = 0;
// Allow up to 20 * 60s for any action to be downloaded from github graph.
int timeoutSeconds = 20 * 60;
while (retryCount < 3)
{
using (var actionDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
using (var actionDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(actionDownloadTimeout.Token, executionContext.CancellationToken))
{
try
{
//open zip stream in async mode
using (FileStream fs = new FileStream(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true))
using (var httpClientHandler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(httpClientHandler))
{
var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN");
if (string.IsNullOrEmpty(authToken))
{
// TODO: Depreciate the PREVIEW_ACTION_TOKEN
authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN");
}
if (!string.IsNullOrEmpty(authToken))
{
HostContext.SecretMasker.AddValue(authToken);
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"PAT:{authToken}"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
}
else
{
var accessToken = executionContext.GetGitHubContext("token");
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{accessToken}"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", base64EncodingToken);
}
httpClient.DefaultRequestHeaders.UserAgent.Add(HostContext.UserAgent);
using (var result = await httpClient.GetStreamAsync(archiveLink))
{
await result.CopyToAsync(fs, _defaultCopyBufferSize, actionDownloadCancellation.Token);
await fs.FlushAsync(actionDownloadCancellation.Token);
// download succeed, break out the retry loop.
break;
}
}
}
catch (OperationCanceledException) when(executionContext.CancellationToken.IsCancellationRequested)
{
Trace.Info($"Action download has been cancelled.");
throw;
}
catch (Exception ex) when(retryCount < 2)
{
retryCount++;
Trace.Error($"Fail to download archive '{archiveLink}' -- Attempt: {retryCount}");
Trace.Error(ex);
if (actionDownloadTimeout.Token.IsCancellationRequested)
{
// action download didn't finish within timeout
executionContext.Warning($"Action '{archiveLink}' didn't finish download within {timeoutSeconds} seconds.");
}
else
{
executionContext.Warning($"Failed to download action '{archiveLink}'. Error {ex.Message}");
}
}
}
if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("_GITHUB_ACTION_DOWNLOAD_NO_BACKOFF")))
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30));
executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
ArgUtil.NotNullOrEmpty(archiveFile, nameof(archiveFile));
executionContext.Debug($"Download '{archiveLink}' to '{archiveFile}'");
var stagingDirectory = Path.Combine(tempDirectory, "_staging");
Directory.CreateDirectory(stagingDirectory);
#if OS_WINDOWS
ZipFile.ExtractToDirectory(archiveFile, stagingDirectory);
#else
string tar = WhichUtil.Which("tar", require: true, trace: Trace);
// tar -xzf
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Info(args.Data);
}
});
processInvoker.ErrorDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Error(args.Data);
}
});
int exitCode = await processInvoker.ExecuteAsync(stagingDirectory, tar, $"-xzf \"{archiveFile}\"", null, executionContext.CancellationToken);
if (exitCode != 0)
{
throw new NotSupportedException($"Can't use 'tar -xzf' extract archive file: {archiveFile}. return code: {exitCode}.");
}
}
#endif
// repository archive from github always contains a nested folder
var subDirectories = new DirectoryInfo(stagingDirectory).GetDirectories();
if (subDirectories.Length != 1)
{
throw new InvalidOperationException($"'{archiveFile}' contains '{subDirectories.Length}' directories");
}
else
{
executionContext.Debug($"Unwrap '{subDirectories[0].Name}' to '{destDirectory}'");
IOUtil.CopyDirectory(subDirectories[0].FullName, destDirectory, executionContext.CancellationToken);
}
Trace.Verbose("Create watermark file indicate action download succeed.");
File.WriteAllText(destDirectory + ".completed", DateTime.UtcNow.ToString());
executionContext.Debug($"Archive '{archiveFile}' has been unzipped into '{destDirectory}'.");
Trace.Info("Finished getting action repository.");
}
finally
{
try
{
//if the temp folder wasn't moved -> wipe it
if (Directory.Exists(tempDirectory))
{
Trace.Verbose("Deleting action temp folder: {0}", tempDirectory);
IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast.
}
}
catch (Exception ex)
{
//it is not critical if we fail to delete the temp folder
Trace.Warning("Failed to delete temp folder '{0}'. Exception: {1}", tempDirectory, ex);
}
}
}
/// <summary>
/// _work
/// \_update
/// \bin
/// \externals
/// \run.sh
/// \run.cmd
/// \package.zip //temp download .zip/.tar.gz
/// </summary>
/// <param name="token"></param>
/// <returns></returns>
private async Task DownloadLatestAgent(CancellationToken token)
{
string latestAgentDirectory = IOUtil.GetUpdatePath(HostContext);
IOUtil.DeleteDirectory(latestAgentDirectory, token);
Directory.CreateDirectory(latestAgentDirectory);
string archiveFile;
if (_targetPackage.Platform.StartsWith("win"))
{
archiveFile = Path.Combine(latestAgentDirectory, "agent.zip");
}
else
{
archiveFile = Path.Combine(latestAgentDirectory, "agent.tar.gz");
}
Trace.Info($"Save latest agent into {archiveFile}.");
try
{
using (var httpClient = new HttpClient(HostContext.CreateHttpClientHandler()))
{
//open zip stream in async mode
using (FileStream fs = new FileStream(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 4096, useAsync: true))
{
using (Stream result = await httpClient.GetStreamAsync(_targetPackage.DownloadUrl))
{
//81920 is the default used by System.IO.Stream.CopyTo and is under the large object heap threshold (85k).
await result.CopyToAsync(fs, 81920, token);
await fs.FlushAsync(token);
}
}
}
if (archiveFile.EndsWith(".zip", StringComparison.OrdinalIgnoreCase))
{
ZipFile.ExtractToDirectory(archiveFile, latestAgentDirectory);
}
else if (archiveFile.EndsWith(".tar.gz", StringComparison.OrdinalIgnoreCase))
{
var whichUtil = HostContext.GetService <IWhichUtil>();
string tar = whichUtil.Which("tar");
if (string.IsNullOrEmpty(tar))
{
throw new NotSupportedException($"tar -xzf");
}
// tar -xzf
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Info(args.Data);
}
});
processInvoker.ErrorDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Error(args.Data);
}
});
int exitCode = await processInvoker.ExecuteAsync(latestAgentDirectory, tar, $"-xzf \"{archiveFile}\"", null, token);
if (exitCode != 0)
{
throw new NotSupportedException($"Can't use 'tar -xzf' extract archive file: {archiveFile}. return code: {exitCode}.");
}
}
}
else
{
throw new NotSupportedException($"{archiveFile}");
}
Trace.Info($"Finished getting latest agent package at: {latestAgentDirectory}.");
}
finally
{
try
{
// delete .zip file
if (!string.IsNullOrEmpty(archiveFile) && File.Exists(archiveFile))
{
Trace.Verbose("Deleting latest agent package zip: {0}", archiveFile);
IOUtil.DeleteFile(archiveFile);
}
}
catch (Exception ex)
{
//it is not critical if we fail to delete the temp folder
Trace.Warning("Failed to delete agent package zip '{0}'. Exception: {1}", archiveFile, ex);
}
}
// copy latest agent into agent root folder
// copy bin from _work/_update -> bin.version under root
string binVersionDir = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"{Constants.Path.BinDirectory}.{_targetPackage.Version}");
Directory.CreateDirectory(binVersionDir);
Trace.Info($"Copy {Path.Combine(latestAgentDirectory, Constants.Path.BinDirectory)} to {binVersionDir}.");
IOUtil.CopyDirectory(Path.Combine(latestAgentDirectory, Constants.Path.BinDirectory), binVersionDir, token);
// copy externals from _work/_update -> externals.version under root
string externalsVersionDir = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"{Constants.Path.ExternalsDirectory}.{_targetPackage.Version}");
Directory.CreateDirectory(externalsVersionDir);
Trace.Info($"Copy {Path.Combine(latestAgentDirectory, Constants.Path.ExternalsDirectory)} to {externalsVersionDir}.");
IOUtil.CopyDirectory(Path.Combine(latestAgentDirectory, Constants.Path.ExternalsDirectory), externalsVersionDir, token);
// copy and replace all .sh/.cmd files
Trace.Info($"Copy any remaining .sh/.cmd files into agent root.");
foreach (FileInfo file in new DirectoryInfo(latestAgentDirectory).GetFiles() ?? new FileInfo[0])
{
// Copy and replace the file.
file.CopyTo(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), file.Name), true);
}
// for windows service back compat with old windows agent, we need make sure the servicehost.exe is still the old name
// if the current bin folder has VsoAgentService.exe, then the new agent bin folder needs VsoAgentService.exe as well
#if OS_WINDOWS
if (File.Exists(Path.Combine(IOUtil.GetBinPath(), "VsoAgentService.exe")))
{
Trace.Info($"Make a copy of AgentService.exe, name it VsoAgentService.exe");
File.Copy(Path.Combine(binVersionDir, "AgentService.exe"), Path.Combine(binVersionDir, "VsoAgentService.exe"), true);
File.Copy(Path.Combine(binVersionDir, "AgentService.exe.config"), Path.Combine(binVersionDir, "VsoAgentService.exe.config"), true);
Trace.Info($"Make a copy of Agent.Listener.exe, name it VsoAgent.exe");
File.Copy(Path.Combine(binVersionDir, "Agent.Listener.exe"), Path.Combine(binVersionDir, "VsoAgent.exe"), true);
File.Copy(Path.Combine(binVersionDir, "Agent.Listener.dll"), Path.Combine(binVersionDir, "VsoAgent.dll"), true);
// in case of we remove all pdb file from agent package.
if (File.Exists(Path.Combine(binVersionDir, "AgentService.pdb")))
{
File.Copy(Path.Combine(binVersionDir, "AgentService.pdb"), Path.Combine(binVersionDir, "VsoAgentService.pdb"), true);
}
if (File.Exists(Path.Combine(binVersionDir, "Agent.Listener.pdb")))
{
File.Copy(Path.Combine(binVersionDir, "Agent.Listener.pdb"), Path.Combine(binVersionDir, "VsoAgent.pdb"), true);
}
}
#endif
}
/// <summary>
/// _work
/// \_update
/// \bin
/// \externals
/// \run.sh
/// \run.cmd
/// \package.zip //temp download .zip/.tar.gz
/// </summary>
/// <param name="token"></param>
/// <returns></returns>
private async Task DownloadLatestAgent(CancellationToken token)
{
string latestAgentDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), Constants.Path.UpdateDirectory);
IOUtil.DeleteDirectory(latestAgentDirectory, token);
Directory.CreateDirectory(latestAgentDirectory);
int agentSuffix = 1;
string archiveFile = null;
bool downloadSucceeded = false;
bool validationSucceeded = false;
try
{
// Download the agent, using multiple attempts in order to be resilient against any networking/CDN issues
for (int attempt = 1; attempt <= Constants.AgentDownloadRetryMaxAttempts && !validationSucceeded; attempt++)
{
// Generate an available package name, and do our best effort to clean up stale local zip files
while (true)
{
if (_targetPackage.Platform.StartsWith("win"))
{
archiveFile = Path.Combine(latestAgentDirectory, $"agent{agentSuffix}.zip");
}
else
{
archiveFile = Path.Combine(latestAgentDirectory, $"agent{agentSuffix}.tar.gz");
}
// The package name is generated, check if there is already a file with the same name and path
if (!string.IsNullOrEmpty(archiveFile) && File.Exists(archiveFile))
{
Trace.Verbose("Deleting latest agent package zip '{0}'", archiveFile);
try
{
// Such a file already exists, so try deleting it
IOUtil.DeleteFile(archiveFile);
// The file was successfully deleted, so we can use the generated package name
break;
}
catch (Exception ex)
{
// Couldn't delete the file for whatever reason, so generate another package name
Trace.Warning("Failed to delete agent package zip '{0}'. Exception: {1}", archiveFile, ex);
agentSuffix++;
}
}
else
{
// There is no a file with the same name and path, so we can use the generated package name
break;
}
}
// Allow a 15-minute package download timeout, which is good enough to update the agent from a 1 Mbit/s ADSL connection.
var timeoutSeconds = AgentKnobs.AgentDownloadTimeout.GetValue(_knobContext).AsInt();
Trace.Info($"Attempt {attempt}: save latest agent into {archiveFile}.");
using (var downloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
using (var downloadCts = CancellationTokenSource.CreateLinkedTokenSource(downloadTimeout.Token, token))
{
try
{
Trace.Info($"Download agent: begin download");
//open zip stream in async mode
using (var handler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(handler))
using (var fs = new FileStream(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 4096, useAsync: true))
using (var result = await httpClient.GetStreamAsync(_targetPackage.DownloadUrl))
{
//81920 is the default used by System.IO.Stream.CopyTo and is under the large object heap threshold (85k).
await result.CopyToAsync(fs, 81920, downloadCts.Token);
await fs.FlushAsync(downloadCts.Token);
}
Trace.Info($"Download agent: finished download");
downloadSucceeded = true;
validationSucceeded = HashValidation(archiveFile);
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
Trace.Info($"Agent download has been canceled.");
throw;
}
catch (SocketException ex)
{
ExceptionsUtil.HandleSocketException(ex, _targetPackage.DownloadUrl, Trace.Warning);
}
catch (Exception ex)
{
if (downloadCts.Token.IsCancellationRequested)
{
Trace.Warning($"Agent download has timed out after {timeoutSeconds} seconds");
}
Trace.Warning($"Failed to get package '{archiveFile}' from '{_targetPackage.DownloadUrl}'. Exception {ex}");
}
}
}
if (!downloadSucceeded)
{
throw new TaskCanceledException($"Agent package '{archiveFile}' failed after {Constants.AgentDownloadRetryMaxAttempts} download attempts.");
}
if (!validationSucceeded)
{
throw new TaskCanceledException(@"Agent package checksum validation failed.
There are possible reasons why this happened:
1) The agent package was compromised.
2) The agent package was not fully downloaded or was corrupted during the download process.
You can skip checksum validation for the agent package by setting the environment variable DISABLE_HASH_VALIDATION=true");
}
// If we got this far, we know that we've successfully downloadeded the agent package
if (archiveFile.EndsWith(".zip", StringComparison.OrdinalIgnoreCase))
{
ZipFile.ExtractToDirectory(archiveFile, latestAgentDirectory);
}
else if (archiveFile.EndsWith(".tar.gz", StringComparison.OrdinalIgnoreCase))
{
string tar = WhichUtil.Which("tar", trace: Trace);
if (string.IsNullOrEmpty(tar))
{
throw new NotSupportedException($"tar -xzf");
}
// tar -xzf
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Info(args.Data);
}
});
processInvoker.ErrorDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Error(args.Data);
}
});
int exitCode = await processInvoker.ExecuteAsync(latestAgentDirectory, tar, $"-xzf \"{archiveFile}\"", null, token);
if (exitCode != 0)
{
throw new NotSupportedException($"Can't use 'tar -xzf' extract archive file: {archiveFile}. return code: {exitCode}.");
}
}
}
else
{
throw new NotSupportedException($"{archiveFile}");
}
Trace.Info($"Finished getting latest agent package at: {latestAgentDirectory}.");
}
finally
{
try
{
// delete .zip file
if (!string.IsNullOrEmpty(archiveFile) && File.Exists(archiveFile))
{
Trace.Verbose("Deleting latest agent package zip: {0}", archiveFile);
IOUtil.DeleteFile(archiveFile);
}
}
catch (Exception ex)
{
//it is not critical if we fail to delete the .zip file
Trace.Warning("Failed to delete agent package zip '{0}'. Exception: {1}", archiveFile, ex);
}
}
if (!String.IsNullOrEmpty(AgentKnobs.DisableAuthenticodeValidation.GetValue(HostContext).AsString()))
{
Trace.Warning("Authenticode validation skipped for downloaded agent package since it is disabled currently by agent settings.");
}
else
{
if (PlatformUtil.RunningOnWindows)
{
var isValid = this.VerifyAgentAuthenticode(latestAgentDirectory);
if (!isValid)
{
throw new Exception("Authenticode validation of agent assemblies failed.");
}
else
{
Trace.Info("Authenticode validation of agent assemblies passed successfully.");
}
}
else
{
Trace.Info("Authenticode validation skipped since it's not supported on non-Windows platforms at the moment.");
}
}
// copy latest agent into agent root folder
// copy bin from _work/_update -> bin.version under root
string binVersionDir = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"{Constants.Path.BinDirectory}.{_targetPackage.Version}");
Directory.CreateDirectory(binVersionDir);
Trace.Info($"Copy {Path.Combine(latestAgentDirectory, Constants.Path.BinDirectory)} to {binVersionDir}.");
IOUtil.CopyDirectory(Path.Combine(latestAgentDirectory, Constants.Path.BinDirectory), binVersionDir, token);
// copy externals from _work/_update -> externals.version under root
string externalsVersionDir = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), $"{Constants.Path.ExternalsDirectory}.{_targetPackage.Version}");
Directory.CreateDirectory(externalsVersionDir);
Trace.Info($"Copy {Path.Combine(latestAgentDirectory, Constants.Path.ExternalsDirectory)} to {externalsVersionDir}.");
IOUtil.CopyDirectory(Path.Combine(latestAgentDirectory, Constants.Path.ExternalsDirectory), externalsVersionDir, token);
// copy and replace all .sh/.cmd files
Trace.Info($"Copy any remaining .sh/.cmd files into agent root.");
foreach (FileInfo file in new DirectoryInfo(latestAgentDirectory).GetFiles() ?? new FileInfo[0])
{
// Copy and replace the file.
file.CopyTo(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), file.Name), true);
}
// for windows service back compat with old windows agent, we need make sure the servicehost.exe is still the old name
// if the current bin folder has VsoAgentService.exe, then the new agent bin folder needs VsoAgentService.exe as well
if (PlatformUtil.RunningOnWindows)
{
if (File.Exists(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), "VsoAgentService.exe")))
{
Trace.Info($"Make a copy of AgentService.exe, name it VsoAgentService.exe");
File.Copy(Path.Combine(binVersionDir, "AgentService.exe"), Path.Combine(binVersionDir, "VsoAgentService.exe"), true);
File.Copy(Path.Combine(binVersionDir, "AgentService.exe.config"), Path.Combine(binVersionDir, "VsoAgentService.exe.config"), true);
Trace.Info($"Make a copy of Agent.Listener.exe, name it VsoAgent.exe");
File.Copy(Path.Combine(binVersionDir, "Agent.Listener.exe"), Path.Combine(binVersionDir, "VsoAgent.exe"), true);
File.Copy(Path.Combine(binVersionDir, "Agent.Listener.dll"), Path.Combine(binVersionDir, "VsoAgent.dll"), true);
// in case of we remove all pdb file from agent package.
if (File.Exists(Path.Combine(binVersionDir, "AgentService.pdb")))
{
File.Copy(Path.Combine(binVersionDir, "AgentService.pdb"), Path.Combine(binVersionDir, "VsoAgentService.pdb"), true);
}
if (File.Exists(Path.Combine(binVersionDir, "Agent.Listener.pdb")))
{
File.Copy(Path.Combine(binVersionDir, "Agent.Listener.pdb"), Path.Combine(binVersionDir, "VsoAgent.pdb"), true);
}
}
}
}
private async Task <string> DownLoadRunner(string downloadDirectory, string packageDownloadUrl, string packageHashValue, CancellationToken token)
{
var stopWatch = Stopwatch.StartNew();
int runnerSuffix = 1;
string archiveFile = null;
bool downloadSucceeded = false;
// Download the runner, using multiple attempts in order to be resilient against any networking/CDN issues
for (int attempt = 1; attempt <= Constants.RunnerDownloadRetryMaxAttempts; attempt++)
{
// Generate an available package name, and do our best effort to clean up stale local zip files
while (true)
{
if (_targetPackage.Platform.StartsWith("win"))
{
archiveFile = Path.Combine(downloadDirectory, $"runner{runnerSuffix}.zip");
}
else
{
archiveFile = Path.Combine(downloadDirectory, $"runner{runnerSuffix}.tar.gz");
}
try
{
// delete .zip file
if (!string.IsNullOrEmpty(archiveFile) && File.Exists(archiveFile))
{
Trace.Verbose("Deleting latest runner package zip '{0}'", archiveFile);
IOUtil.DeleteFile(archiveFile);
}
break;
}
catch (Exception ex)
{
// couldn't delete the file for whatever reason, so generate another name
Trace.Warning("Failed to delete runner package zip '{0}'. Exception: {1}", archiveFile, ex);
runnerSuffix++;
}
}
// Allow a 15-minute package download timeout, which is good enough to update the runner from a 1 Mbit/s ADSL connection.
if (!int.TryParse(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_DOWNLOAD_TIMEOUT") ?? string.Empty, out int timeoutSeconds))
{
timeoutSeconds = 15 * 60;
}
Trace.Info($"Attempt {attempt}: save latest runner into {archiveFile}.");
using (var downloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
using (var downloadCts = CancellationTokenSource.CreateLinkedTokenSource(downloadTimeout.Token, token))
{
try
{
Trace.Info($"Download runner: begin download");
long downloadSize = 0;
//open zip stream in async mode
using (HttpClient httpClient = new HttpClient(HostContext.CreateHttpClientHandler()))
{
if (!string.IsNullOrEmpty(_targetPackage.Token))
{
Trace.Info($"Adding authorization token ({_targetPackage.Token.Length} chars)");
httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", _targetPackage.Token);
}
Trace.Info($"Downloading {packageDownloadUrl}");
using (FileStream fs = new FileStream(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 4096, useAsync: true))
using (Stream result = await httpClient.GetStreamAsync(packageDownloadUrl))
{
//81920 is the default used by System.IO.Stream.CopyTo and is under the large object heap threshold (85k).
await result.CopyToAsync(fs, 81920, downloadCts.Token);
await fs.FlushAsync(downloadCts.Token);
downloadSize = fs.Length;
}
}
Trace.Info($"Download runner: finished download");
downloadSucceeded = true;
stopWatch.Stop();
_updateTrace.Add($"PackageDownloadTime: {stopWatch.ElapsedMilliseconds}ms");
_updateTrace.Add($"Attempts: {attempt}");
_updateTrace.Add($"PackageSize: {downloadSize / 1024 / 1024}MB");
break;
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
Trace.Info($"Runner download has been canceled.");
throw;
}
catch (Exception ex)
{
if (downloadCts.Token.IsCancellationRequested)
{
Trace.Warning($"Runner download has timed out after {timeoutSeconds} seconds");
}
Trace.Warning($"Failed to get package '{archiveFile}' from '{packageDownloadUrl}'. Exception {ex}");
}
}
}
if (downloadSucceeded)
{
return(archiveFile);
}
else
{
return(null);
}
}
private async Task <GitHubRunnerRegisterToken> GetJITRunnerTokenAsync(string githubUrl, string githubToken, string tokenType)
{
var githubApiUrl = "";
var gitHubUrlBuilder = new UriBuilder(githubUrl);
var path = gitHubUrlBuilder.Path.Split('/', '\\', StringSplitOptions.RemoveEmptyEntries);
if (path.Length == 1)
{
// org runner
if (UrlUtil.IsHostedServer(gitHubUrlBuilder))
{
githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/orgs/{path[0]}/actions/runners/{tokenType}-token";
}
else
{
githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/orgs/{path[0]}/actions/runners/{tokenType}-token";
}
}
else if (path.Length == 2)
{
// repo or enterprise runner.
var repoScope = "repos/";
if (string.Equals(path[0], "enterprises", StringComparison.OrdinalIgnoreCase))
{
repoScope = "";
}
if (UrlUtil.IsHostedServer(gitHubUrlBuilder))
{
githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/{repoScope}{path[0]}/{path[1]}/actions/runners/{tokenType}-token";
}
else
{
githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/{repoScope}{path[0]}/{path[1]}/actions/runners/{tokenType}-token";
}
}
else
{
throw new ArgumentException($"'{githubUrl}' should point to an org or repository.");
}
using (var httpClientHandler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(httpClientHandler))
{
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"github:{githubToken}"));
HostContext.SecretMasker.AddValue(base64EncodingToken);
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("basic", base64EncodingToken);
httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents);
httpClient.DefaultRequestHeaders.Accept.ParseAdd("application/vnd.github.v3+json");
var response = await httpClient.PostAsync(githubApiUrl, new StringContent(string.Empty));
if (response.IsSuccessStatusCode)
{
Trace.Info($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
var jsonResponse = await response.Content.ReadAsStringAsync();
return(StringUtil.ConvertFromJson <GitHubRunnerRegisterToken>(jsonResponse));
}
else
{
_term.WriteError($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
var errorResponse = await response.Content.ReadAsStringAsync();
_term.WriteError(errorResponse);
response.EnsureSuccessStatusCode();
return(null);
}
}
}
public async Task DownloadAsync(
IExecutionContext executionContext,
ArtifactDefinition artifactDefinition,
string localFolderPath)
{
ArgUtil.NotNull(artifactDefinition, nameof(artifactDefinition));
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNullOrEmpty(localFolderPath, nameof(localFolderPath));
var jenkinsDetails = artifactDefinition.Details as JenkinsArtifactDetails;
executionContext.Output(StringUtil.Loc("RMGotJenkinsArtifactDetails"));
executionContext.Output(StringUtil.Loc("RMJenkinsJobName", jenkinsDetails.JobName));
executionContext.Output(StringUtil.Loc("RMJenkinsBuildId", jenkinsDetails.BuildId));
Stream downloadedStream = null;
using (HttpClientHandler handler = HostContext.CreateHttpClientHandler())
{
handler.ServerCertificateCustomValidationCallback = (message, certificate, chain, sslPolicyErrors) =>
{
return(jenkinsDetails.AcceptUntrustedCertificates);
};
using (HttpClient client = new HttpClient(handler))
{
SetupHttpClient(client, jenkinsDetails.AccountName, jenkinsDetails.AccountPassword);
if (!IsValidBuild(client, jenkinsDetails))
{
throw new ArtifactDownloadException(StringUtil.Loc("RMJenkinsInvalidBuild", jenkinsDetails.BuildId));
}
var downloadArtifactsUrl =
new Uri(
string.Format(
CultureInfo.InvariantCulture,
"{0}/job/{1}/{2}/artifact/{3}/*zip*/",
jenkinsDetails.Url,
jenkinsDetails.JobName,
jenkinsDetails.BuildId,
jenkinsDetails.RelativePath));
executionContext.Output(StringUtil.Loc("RMPrepareToGetFromJenkinsServer"));
HttpResponseMessage response = client.GetAsync(downloadArtifactsUrl).Result;
if (response.IsSuccessStatusCode)
{
downloadedStream = response.Content.ReadAsStreamAsync().Result;
}
else if (response.StatusCode == HttpStatusCode.NotFound)
{
executionContext.Warning(StringUtil.Loc("RMJenkinsNoArtifactsFound", jenkinsDetails.BuildId));
return;
}
else
{
throw new ArtifactDownloadException(StringUtil.Loc("RMDownloadArtifactUnexpectedError"));
}
}
}
var parentFolder = GetParentFolderName(jenkinsDetails.RelativePath);
Trace.Info($"Found parentFolder {parentFolder} for relative path {jenkinsDetails.RelativePath}");
executionContext.Output(StringUtil.Loc("RMDownloadingJenkinsArtifacts"));
var zipStreamDownloader = HostContext.GetService <IZipStreamDownloader>();
await zipStreamDownloader.DownloadFromStream(
executionContext,
downloadedStream,
string.IsNullOrEmpty(parentFolder)? "archive" : string.Empty,
parentFolder,
localFolderPath);
}
private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, ActionDownloadDetails actionDownloadDetails, string destDirectory)
{
//download and extract action in a temp folder and rename it on success
string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), "_temp_" + Guid.NewGuid());
Directory.CreateDirectory(tempDirectory);
#if OS_WINDOWS
string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.zip");
#else
string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.tar.gz");
#endif
string link = actionDownloadDetails.ArchiveLink;
Trace.Info($"Save archive '{link}' into {archiveFile}.");
try
{
int retryCount = 0;
// Allow up to 20 * 60s for any action to be downloaded from github graph.
int timeoutSeconds = 20 * 60;
while (retryCount < 3)
{
using (var actionDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)))
using (var actionDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(actionDownloadTimeout.Token, executionContext.CancellationToken))
{
try
{
//open zip stream in async mode
using (FileStream fs = new FileStream(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true))
using (var httpClientHandler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(httpClientHandler))
{
actionDownloadDetails.ConfigureAuthorization(executionContext, httpClient);
httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents);
using (var response = await httpClient.GetAsync(link))
{
if (response.IsSuccessStatusCode)
{
using (var result = await response.Content.ReadAsStreamAsync())
{
await result.CopyToAsync(fs, _defaultCopyBufferSize, actionDownloadCancellation.Token);
await fs.FlushAsync(actionDownloadCancellation.Token);
// download succeed, break out the retry loop.
break;
}
}
else if (response.StatusCode == HttpStatusCode.NotFound)
{
// It doesn't make sense to retry in this case, so just stop
throw new ActionNotFoundException(new Uri(link));
}
else
{
// Something else bad happened, let's go to our retry logic
response.EnsureSuccessStatusCode();
}
}
}
}
catch (OperationCanceledException) when(executionContext.CancellationToken.IsCancellationRequested)
{
Trace.Info("Action download has been cancelled.");
throw;
}
catch (ActionNotFoundException)
{
Trace.Info($"The action at '{link}' does not exist");
throw;
}
catch (Exception ex) when(retryCount < 2)
{
retryCount++;
Trace.Error($"Fail to download archive '{link}' -- Attempt: {retryCount}");
Trace.Error(ex);
if (actionDownloadTimeout.Token.IsCancellationRequested)
{
// action download didn't finish within timeout
executionContext.Warning($"Action '{link}' didn't finish download within {timeoutSeconds} seconds.");
}
else
{
executionContext.Warning($"Failed to download action '{link}'. Error: {ex.Message}");
}
}
}
if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("_GITHUB_ACTION_DOWNLOAD_NO_BACKOFF")))
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30));
executionContext.Warning($"Back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
ArgUtil.NotNullOrEmpty(archiveFile, nameof(archiveFile));
executionContext.Debug($"Download '{link}' to '{archiveFile}'");
var stagingDirectory = Path.Combine(tempDirectory, "_staging");
Directory.CreateDirectory(stagingDirectory);
#if OS_WINDOWS
ZipFile.ExtractToDirectory(archiveFile, stagingDirectory);
#else
string tar = WhichUtil.Which("tar", require: true, trace: Trace);
// tar -xzf
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Info(args.Data);
}
});
processInvoker.ErrorDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
Trace.Error(args.Data);
}
});
int exitCode = await processInvoker.ExecuteAsync(stagingDirectory, tar, $"-xzf \"{archiveFile}\"", null, executionContext.CancellationToken);
if (exitCode != 0)
{
throw new NotSupportedException($"Can't use 'tar -xzf' extract archive file: {archiveFile}. return code: {exitCode}.");
}
}
#endif
// repository archive from github always contains a nested folder
var subDirectories = new DirectoryInfo(stagingDirectory).GetDirectories();
if (subDirectories.Length != 1)
{
throw new InvalidOperationException($"'{archiveFile}' contains '{subDirectories.Length}' directories");
}
else
{
executionContext.Debug($"Unwrap '{subDirectories[0].Name}' to '{destDirectory}'");
IOUtil.CopyDirectory(subDirectories[0].FullName, destDirectory, executionContext.CancellationToken);
}
Trace.Verbose("Create watermark file indicate action download succeed.");
string watermarkFile = GetWatermarkFilePath(destDirectory);
File.WriteAllText(watermarkFile, DateTime.UtcNow.ToString());
executionContext.Debug($"Archive '{archiveFile}' has been unzipped into '{destDirectory}'.");
Trace.Info("Finished getting action repository.");
}
finally
{
try
{
//if the temp folder wasn't moved -> wipe it
if (Directory.Exists(tempDirectory))
{
Trace.Verbose("Deleting action temp folder: {0}", tempDirectory);
IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast.
}
}
catch (Exception ex)
{
//it is not critical if we fail to delete the temp folder
Trace.Warning("Failed to delete temp folder '{0}'. Exception: {1}", tempDirectory, ex);
}
}
}