public void SetupProxy(string proxyUrl, string proxyUsername, string proxyPassword)
{
if (!string.IsNullOrEmpty(proxyUrl))
{
Uri proxy = UrlUtil.GetCredentialEmbeddedUrl(new Uri(proxyUrl), proxyUsername, proxyPassword);
AdditionalEnvironmentVariables["http_proxy"] = proxy.AbsoluteUri;
}
}
public void GetCredentialEmbeddedUrl_UsernameAndPasswordEncoding()
{
// Act.
Uri result = UrlUtil.GetCredentialEmbeddedUrl(new Uri("https://github.com/Microsoft/vsts-agent.git"), "user 123", "password 123");
// Actual
Assert.Equal("https://user%20123:password%[email protected]/Microsoft/vsts-agent.git", result.AbsoluteUri);
}
public void GetCredentialEmbeddedUrl_NoPassword()
{
// Act.
Uri result = UrlUtil.GetCredentialEmbeddedUrl(new Uri("https://github.com/Microsoft/vsts-agent.git"), "user123", string.Empty);
// Actual
Assert.Equal("https://[email protected]/Microsoft/vsts-agent.git", result.AbsoluteUri);
}
public void GetCredentialEmbeddedUrl_NoUsername()
{
// Act.
Uri result = UrlUtil.GetCredentialEmbeddedUrl(new Uri("https://github.com/Microsoft/vsts-agent.git"), string.Empty, "password123");
// Actual
Assert.Equal("https://*****:*****@github.com/Microsoft/vsts-agent.git", result.AbsoluteUri);
}
public void GetCredentialEmbeddedUrl_HasUsernameAndPassword()
{
// Act.
Uri result = UrlUtil.GetCredentialEmbeddedUrl(new Uri("https://github.com/actions/runner.git"), "user123", "password123");
// Actual
Assert.Equal("https://*****:*****@github.com/actions/runner.git", result.AbsoluteUri);
}
public void GetCredentialEmbeddedUrl_NoUsernameAndPassword()
{
// Act.
Uri result = UrlUtil.GetCredentialEmbeddedUrl(new Uri("https://github.com/actions/runner.git"), string.Empty, string.Empty);
// Actual
Assert.Equal("https://github.com/actions/runner.git", result.AbsoluteUri);
}
public async Task GetSourceAsync(
IExecutionContext executionContext,
ServiceEndpoint endpoint,
CancellationToken cancellationToken)
{
Trace.Entering();
// Validate args.
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(endpoint, nameof(endpoint));
executionContext.Output($"Syncing repository: {endpoint.Name} ({RepositoryType})");
Uri repositoryUrl = endpoint.Url;
if (!repositoryUrl.IsAbsoluteUri)
{
throw new InvalidOperationException("Repository url need to be an absolute uri.");
}
string targetPath = GetEndpointData(endpoint, Constants.EndpointData.SourcesDirectory);
string sourceBranch = GetEndpointData(endpoint, Constants.EndpointData.SourceBranch);
string sourceVersion = GetEndpointData(endpoint, Constants.EndpointData.SourceVersion);
bool clean = false;
if (endpoint.Data.ContainsKey(WellKnownEndpointData.Clean))
{
clean = StringUtil.ConvertToBoolean(endpoint.Data[WellKnownEndpointData.Clean]);
}
bool checkoutSubmodules = false;
if (endpoint.Data.ContainsKey(WellKnownEndpointData.CheckoutSubmodules))
{
checkoutSubmodules = StringUtil.ConvertToBoolean(endpoint.Data[WellKnownEndpointData.CheckoutSubmodules]);
}
bool exposeCred = executionContext.Variables.GetBoolean(Constants.Variables.System.EnableAccessToken) ?? false;
Trace.Info($"Repository url={repositoryUrl}");
Trace.Info($"targetPath={targetPath}");
Trace.Info($"sourceBranch={sourceBranch}");
Trace.Info($"sourceVersion={sourceVersion}");
Trace.Info($"clean={clean}");
Trace.Info($"checkoutSubmodules={checkoutSubmodules}");
Trace.Info($"exposeCred={exposeCred}");
// Determine which git will be use
// On windows, we prefer the built-in portable git within the agent's externals folder, set system.prefergitfrompath=true can change the behavior,
// agent will find git.exe from %PATH%
// On Linux, we will always use git find in %PATH% regardless of system.prefergitfrompath
bool preferGitFromPath = executionContext.Variables.GetBoolean(Constants.Variables.System.PreferGitFromPath) ?? false;
#if !OS_WINDOWS
// On linux/OSX, we will always find git from %PATH%
preferGitFromPath = true;
#endif
// Determine do we need to provide creds to git operation
_selfManageGitCreds = executionContext.Variables.GetBoolean(Constants.Variables.System.SelfManageGitCreds) ?? false;
if (_selfManageGitCreds)
{
// Customer choose to own git creds by themselves.
executionContext.Output(StringUtil.Loc("SelfManageGitCreds"));
}
// Initialize git command manager
_gitCommandManager = HostContext.GetService <IGitCommandManager>();
await _gitCommandManager.LoadGitExecutionInfo(executionContext, useBuiltInGit : !preferGitFromPath);
// Make sure the build machine met all requirements for the git repository
// For now, the only requirement we have is git version greater than 2.9 for on-prem tfsgit
RequirementCheck(executionContext, endpoint);
// retrieve credential from endpoint.
string username = string.Empty;
string password = string.Empty;
if (!_selfManageGitCreds && endpoint.Authorization != null)
{
switch (endpoint.Authorization.Scheme)
{
case EndpointAuthorizationSchemes.OAuth:
username = EndpointAuthorizationSchemes.OAuth;
if (!endpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.AccessToken, out password))
{
password = string.Empty;
}
break;
case EndpointAuthorizationSchemes.UsernamePassword:
if (!endpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.Username, out username))
{
// leave the username as empty, the username might in the url, like: http://[email protected]
username = string.Empty;
}
if (!endpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.Password, out password))
{
// we have username, but no password
password = string.Empty;
}
break;
default:
executionContext.Warning($"Unsupport endpoint authorization schemes: {endpoint.Authorization.Scheme}");
break;
}
}
// prepare credentail embedded urls
_repositoryUrlWithCred = UrlUtil.GetCredentialEmbeddedUrl(repositoryUrl, username, password);
if (!string.IsNullOrEmpty(executionContext.Variables.Agent_ProxyUrl))
{
_proxyUrlWithCred = UrlUtil.GetCredentialEmbeddedUrl(new Uri(executionContext.Variables.Agent_ProxyUrl), executionContext.Variables.Agent_ProxyUsername, executionContext.Variables.Agent_ProxyPassword);
}
// Check the current contents of the root folder to see if there is already a repo
// If there is a repo, see if it matches the one we are expecting to be there based on the remote fetch url
// if the repo is not what we expect, remove the folder
if (!await IsRepositoryOriginUrlMatch(executionContext, targetPath, repositoryUrl))
{
// Delete source folder
IOUtil.DeleteDirectory(targetPath, cancellationToken);
}
else
{
// delete the index.lock file left by previous canceled build or any operation casue git.exe crash last time.
string lockFile = Path.Combine(targetPath, ".git\\index.lock");
if (File.Exists(lockFile))
{
try
{
File.Delete(lockFile);
}
catch (Exception ex)
{
executionContext.Debug($"Unable to delete the index.lock file: {lockFile}");
executionContext.Debug(ex.ToString());
}
}
// When repo.clean is selected for a git repo, execute git clean -fdx and git reset --hard HEAD on the current repo.
// This will help us save the time to reclone the entire repo.
// If any git commands exit with non-zero return code or any exception happened during git.exe invoke, fall back to delete the repo folder.
if (clean)
{
Boolean softCleanSucceed = true;
// git clean -fdx
int exitCode_clean = await _gitCommandManager.GitClean(executionContext, targetPath);
if (exitCode_clean != 0)
{
executionContext.Debug($"'git clean -fdx' failed with exit code {exitCode_clean}, this normally caused by:\n 1) Path too long\n 2) Permission issue\n 3) File in use\nFor futher investigation, manually run 'git clean -fdx' on repo root: {targetPath} after each build.");
softCleanSucceed = false;
}
// git reset --hard HEAD
if (softCleanSucceed)
{
int exitCode_reset = await _gitCommandManager.GitReset(executionContext, targetPath);
if (exitCode_reset != 0)
{
executionContext.Debug($"'git reset --hard HEAD' failed with exit code {exitCode_reset}\nFor futher investigation, manually run 'git reset --hard HEAD' on repo root: {targetPath} after each build.");
softCleanSucceed = false;
}
}
// git clean -fdx and git reset --hard HEAD for each submodule
if (checkoutSubmodules)
{
if (softCleanSucceed)
{
int exitCode_submoduleclean = await _gitCommandManager.GitSubmoduleClean(executionContext, targetPath);
if (exitCode_submoduleclean != 0)
{
executionContext.Debug($"'git submodule foreach git clean -fdx' failed with exit code {exitCode_submoduleclean}\nFor futher investigation, manually run 'git submodule foreach git clean -fdx' on repo root: {targetPath} after each build.");
softCleanSucceed = false;
}
}
if (softCleanSucceed)
{
int exitCode_submodulereset = await _gitCommandManager.GitSubmoduleReset(executionContext, targetPath);
if (exitCode_submodulereset != 0)
{
executionContext.Debug($"'git submodule foreach git reset --hard HEAD' failed with exit code {exitCode_submodulereset}\nFor futher investigation, manually run 'git submodule foreach git reset --hard HEAD' on repo root: {targetPath} after each build.");
softCleanSucceed = false;
}
}
}
if (!softCleanSucceed)
{
//fall back
executionContext.Warning("Unable to run \"git clean -fdx\" and \"git reset --hard HEAD\" successfully, delete source folder instead.");
IOUtil.DeleteDirectory(targetPath, cancellationToken);
}
}
}
// if the folder is missing, create it
if (!Directory.Exists(targetPath))
{
Directory.CreateDirectory(targetPath);
}
// if the folder contains a .git folder, it means the folder contains a git repo that matches the remote url and in a clean state.
// we will run git fetch to update the repo.
if (!Directory.Exists(Path.Combine(targetPath, ".git")))
{
// init git repository
int exitCode_init = await _gitCommandManager.GitInit(executionContext, targetPath);
if (exitCode_init != 0)
{
throw new InvalidOperationException($"Unable to use git.exe init repository under {targetPath}, 'git init' failed with exit code: {exitCode_init}");
}
int exitCode_addremote = await _gitCommandManager.GitRemoteAdd(executionContext, targetPath, "origin", repositoryUrl.AbsoluteUri);
if (exitCode_addremote != 0)
{
throw new InvalidOperationException($"Unable to use git.exe add remote 'origin', 'git remote add' failed with exit code: {exitCode_addremote}");
}
}
cancellationToken.ThrowIfCancellationRequested();
executionContext.Progress(0, "Starting fetch...");
// disable git auto gc
int exitCode_disableGC = await _gitCommandManager.GitDisableAutoGC(executionContext, targetPath);
if (exitCode_disableGC != 0)
{
executionContext.Warning("Unable turn off git auto garbage collection, git fetch operation may trigger auto garbage collection which will affect the performence of fetching.");
}
// always remove any possible left extraheader setting from git config.
if (await _gitCommandManager.GitConfigExist(executionContext, targetPath, $"http.{repositoryUrl.AbsoluteUri}.extraheader"))
{
executionContext.Debug("Remove any extraheader setting from git config.");
await RemoveGitConfig(executionContext, targetPath, $"http.{repositoryUrl.AbsoluteUri}.extraheader", string.Empty);
}
// always remove any possible left proxy setting from git config, the proxy setting may contains credential
if (await _gitCommandManager.GitConfigExist(executionContext, targetPath, $"http.proxy"))
{
executionContext.Debug("Remove any proxy setting from git config.");
await RemoveGitConfig(executionContext, targetPath, $"http.proxy", string.Empty);
}
List <string> additionalFetchArgs = new List <string>();
if (!_selfManageGitCreds)
{
// v2.9 git support provide auth header as cmdline arg.
// as long 2.9 git exist, VSTS repo, TFS repo and Github repo will use this to handle auth challenge.
if (UseAuthHeaderCmdlineArg)
{
additionalFetchArgs.Add($"-c http.extraheader=\"AUTHORIZATION: {GenerateAuthHeader(username, password)}\"");
}
else
{
// Otherwise, inject credential into fetch/push url
// inject credential into fetch url
executionContext.Debug("Inject credential into git remote url.");
ArgUtil.NotNull(_repositoryUrlWithCred, nameof(_repositoryUrlWithCred));
// inject credential into fetch url
executionContext.Debug("Inject credential into git remote fetch url.");
int exitCode_seturl = await _gitCommandManager.GitRemoteSetUrl(executionContext, targetPath, "origin", _repositoryUrlWithCred.AbsoluteUri);
if (exitCode_seturl != 0)
{
throw new InvalidOperationException($"Unable to use git.exe inject credential to git remote fetch url, 'git remote set-url' failed with exit code: {exitCode_seturl}");
}
// inject credential into push url
executionContext.Debug("Inject credential into git remote push url.");
exitCode_seturl = await _gitCommandManager.GitRemoteSetPushUrl(executionContext, targetPath, "origin", _repositoryUrlWithCred.AbsoluteUri);
if (exitCode_seturl != 0)
{
throw new InvalidOperationException($"Unable to use git.exe inject credential to git remote push url, 'git remote set-url --push' failed with exit code: {exitCode_seturl}");
}
}
// Prepare proxy config for fetch.
if (!string.IsNullOrEmpty(executionContext.Variables.Agent_ProxyUrl))
{
executionContext.Debug($"Config proxy server '{executionContext.Variables.Agent_ProxyUrl}' for git fetch.");
ArgUtil.NotNull(_proxyUrlWithCred, nameof(_proxyUrlWithCred));
additionalFetchArgs.Add($"-c http.proxy=\"{_proxyUrlWithCred.AbsoluteUri}\"");
}
}
// If this is a build for a pull request, then include
// the pull request reference as an additional ref.
string fetchSpec = IsPullRequest(sourceBranch) ? StringUtil.Format("+{0}:{1}", sourceBranch, GetRemoteRefName(sourceBranch)) : null;
int exitCode_fetch = await _gitCommandManager.GitFetch(executionContext, targetPath, "origin", new List <string>() { fetchSpec }, string.Join(" ", additionalFetchArgs), cancellationToken);
if (exitCode_fetch != 0)
{
throw new InvalidOperationException($"Git fetch failed with exit code: {exitCode_fetch}");
}
// Checkout
// sourceToBuild is used for checkout
// if sourceBranch is a PR branch or sourceVersion is null, make sure branch name is a remote branch. we need checkout to detached head.
// (change refs/heads to refs/remotes/origin, refs/pull to refs/remotes/pull, or leava it as it when the branch name doesn't contain refs/...)
// if sourceVersion provide, just use that for checkout, since when you checkout a commit, it will end up in detached head.
cancellationToken.ThrowIfCancellationRequested();
executionContext.Progress(80, "Starting checkout...");
string sourcesToBuild;
if (IsPullRequest(sourceBranch) || string.IsNullOrEmpty(sourceVersion))
{
sourcesToBuild = GetRemoteRefName(sourceBranch);
}
else
{
sourcesToBuild = sourceVersion;
}
// Finally, checkout the sourcesToBuild (if we didn't find a valid git object this will throw)
int exitCode_checkout = await _gitCommandManager.GitCheckout(executionContext, targetPath, sourcesToBuild, cancellationToken);
if (exitCode_checkout != 0)
{
throw new InvalidOperationException($"Git checkout failed with exit code: {exitCode_checkout}");
}
// Submodule update
if (checkoutSubmodules)
{
cancellationToken.ThrowIfCancellationRequested();
executionContext.Progress(90, "Updating submodules...");
int exitCode_submoduleInit = await _gitCommandManager.GitSubmoduleInit(executionContext, targetPath);
if (exitCode_submoduleInit != 0)
{
throw new InvalidOperationException($"Git submodule init failed with exit code: {exitCode_submoduleInit}");
}
List <string> additionalSubmoduleUpdateArgs = new List <string>();
if (!_selfManageGitCreds)
{
if (UseAuthHeaderCmdlineArg)
{
string authorityUrl = repositoryUrl.AbsoluteUri.Replace(repositoryUrl.PathAndQuery, string.Empty);
additionalSubmoduleUpdateArgs.Add($"-c http.{authorityUrl}.extraheader=\"AUTHORIZATION: {GenerateAuthHeader(username, password)}\"");
}
// Prepare proxy config for submodule update.
if (!string.IsNullOrEmpty(executionContext.Variables.Agent_ProxyUrl))
{
executionContext.Debug($"Config proxy server '{executionContext.Variables.Agent_ProxyUrl}' for git submodule update.");
ArgUtil.NotNull(_proxyUrlWithCred, nameof(_proxyUrlWithCred));
additionalSubmoduleUpdateArgs.Add($"-c http.proxy=\"{_proxyUrlWithCred.AbsoluteUri}\"");
}
}
int exitCode_submoduleUpdate = await _gitCommandManager.GitSubmoduleUpdate(executionContext, targetPath, string.Join(" ", additionalSubmoduleUpdateArgs), cancellationToken);
if (exitCode_submoduleUpdate != 0)
{
throw new InvalidOperationException($"Git submodule update failed with exit code: {exitCode_submoduleUpdate}");
}
}
// handle expose creds, related to 'Allow Scripts to Access OAuth Token' option
if (!_selfManageGitCreds)
{
if (UseAuthHeaderCmdlineArg && exposeCred)
{
string configKey = $"http.{repositoryUrl.AbsoluteUri}.extraheader";
string configValue = $"\"AUTHORIZATION: {GenerateAuthHeader(username, password)}\"";
_configModifications[configKey] = configValue.Trim('\"');
int exitCode_config = await _gitCommandManager.GitConfig(executionContext, targetPath, configKey, configValue);
if (exitCode_config != 0)
{
throw new InvalidOperationException($"Git config failed with exit code: {exitCode_config}");
}
}
if (!UseAuthHeaderCmdlineArg && !exposeCred)
{
// remove cached credential from origin's fetch/push url.
await RemoveCachedCredential(executionContext, targetPath, repositoryUrl, "origin");
}
if (exposeCred)
{
// save proxy setting to git config.
if (!string.IsNullOrEmpty(executionContext.Variables.Agent_ProxyUrl))
{
executionContext.Debug($"Save proxy config for proxy server '{executionContext.Variables.Agent_ProxyUrl}' into git config.");
ArgUtil.NotNull(_proxyUrlWithCred, nameof(_proxyUrlWithCred));
string proxyConfigKey = "http.proxy";
string proxyConfigValue = $"\"{_proxyUrlWithCred.AbsoluteUri}\"";
_configModifications[proxyConfigKey] = proxyConfigValue.Trim('\"');
int exitCode_proxyconfig = await _gitCommandManager.GitConfig(executionContext, targetPath, proxyConfigKey, proxyConfigValue);
if (exitCode_proxyconfig != 0)
{
throw new InvalidOperationException($"Git config failed with exit code: {exitCode_proxyconfig}");
}
}
}
}
}
private async Task <CheckResult> CheckGit(string url, string pat, bool extraCA = false)
{
var result = new CheckResult();
try
{
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** Validate server cert and proxy configuration with Git ");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
var repoUrlBuilder = new UriBuilder(url);
repoUrlBuilder.Path = "actions/checkout";
repoUrlBuilder.UserName = "******";
repoUrlBuilder.Password = pat;
var gitProxy = "";
var proxy = HostContext.WebProxy.GetProxy(repoUrlBuilder.Uri);
if (proxy != null)
{
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} Runner is behind http proxy '{proxy.AbsoluteUri}'");
if (HostContext.WebProxy.HttpProxyUsername != null ||
HostContext.WebProxy.HttpsProxyUsername != null)
{
var proxyUrlWithCred = UrlUtil.GetCredentialEmbeddedUrl(
proxy,
HostContext.WebProxy.HttpProxyUsername ?? HostContext.WebProxy.HttpsProxyUsername,
HostContext.WebProxy.HttpProxyPassword ?? HostContext.WebProxy.HttpsProxyPassword);
gitProxy = $"-c http.proxy={proxyUrlWithCred}";
}
else
{
gitProxy = $"-c http.proxy={proxy.AbsoluteUri}";
}
}
using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
{
processInvoker.OutputDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} {args.Data}");
}
});
processInvoker.ErrorDataReceived += new EventHandler <ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} {args.Data}");
}
});
var gitArgs = $"{gitProxy} ls-remote --exit-code {repoUrlBuilder.Uri.AbsoluteUri} HEAD";
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} Run 'git {gitArgs}' ");
var env = new Dictionary <string, string>
{
{ "GIT_TRACE", "1" },
{ "GIT_CURL_VERBOSE", "1" }
};
if (extraCA)
{
env["GIT_SSL_CAINFO"] = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), "download_ca_cert.pem");
}
await processInvoker.ExecuteAsync(
HostContext.GetDirectory(WellKnownDirectory.Root),
_gitPath,
gitArgs,
env,
true,
CancellationToken.None);
}
result.Pass = true;
}
catch (Exception ex)
{
result.Pass = false;
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** git ls-remote failed with error: {ex}");
if (result.Logs.Any(x => x.Contains("SSL Certificate problem", StringComparison.OrdinalIgnoreCase)))
{
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** git ls-remote failed due to SSL cert issue.");
result.SslError = true;
}
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
}
return(result);
}