public async Task GetSourceAsync( IExecutionContext executionContext, ServiceEndpoint endpoint, CancellationToken cancellationToken) { Trace.Entering(); // Validate args. ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(endpoint, nameof(endpoint)); executionContext.Output($"Syncing repository: {endpoint.Name} ({RepositoryType})"); Uri repositoryUrl = endpoint.Url; if (!repositoryUrl.IsAbsoluteUri) { throw new InvalidOperationException("Repository url need to be an absolute uri."); } string targetPath = GetEndpointData(endpoint, Constants.EndpointData.SourcesDirectory); string sourceBranch = GetEndpointData(endpoint, Constants.EndpointData.SourceBranch); string sourceVersion = GetEndpointData(endpoint, Constants.EndpointData.SourceVersion); bool clean = false; if (endpoint.Data.ContainsKey(WellKnownEndpointData.Clean)) { clean = StringUtil.ConvertToBoolean(endpoint.Data[WellKnownEndpointData.Clean]); } bool checkoutSubmodules = false; if (endpoint.Data.ContainsKey(WellKnownEndpointData.CheckoutSubmodules)) { checkoutSubmodules = StringUtil.ConvertToBoolean(endpoint.Data[WellKnownEndpointData.CheckoutSubmodules]); } bool exposeCred = executionContext.Variables.GetBoolean(Constants.Variables.System.EnableAccessToken) ?? false; Trace.Info($"Repository url={repositoryUrl}"); Trace.Info($"targetPath={targetPath}"); Trace.Info($"sourceBranch={sourceBranch}"); Trace.Info($"sourceVersion={sourceVersion}"); Trace.Info($"clean={clean}"); Trace.Info($"checkoutSubmodules={checkoutSubmodules}"); Trace.Info($"exposeCred={exposeCred}"); // Determine which git will be use // On windows, we prefer the built-in portable git within the agent's externals folder, set system.prefergitfrompath=true can change the behavior, // agent will find git.exe from %PATH% // On Linux, we will always use git find in %PATH% regardless of system.prefergitfrompath bool preferGitFromPath = executionContext.Variables.GetBoolean(Constants.Variables.System.PreferGitFromPath) ?? false; #if !OS_WINDOWS // On linux/OSX, we will always find git from %PATH% preferGitFromPath = true; #endif // Determine do we need to provide creds to git operation _selfManageGitCreds = executionContext.Variables.GetBoolean(Constants.Variables.System.SelfManageGitCreds) ?? false; if (_selfManageGitCreds) { // Customer choose to own git creds by themselves. executionContext.Output(StringUtil.Loc("SelfManageGitCreds")); } // Initialize git command manager _gitCommandManager = HostContext.GetService <IGitCommandManager>(); await _gitCommandManager.LoadGitExecutionInfo(executionContext, useBuiltInGit : !preferGitFromPath); // min git version that support add extra auth header. Version minGitVersionSupportAuthHeader = new Version(2, 9); // When the repository is a TfsGit, figure out the endpoint is hosted vsts git or on-prem tfs git bool?onPremTfsGit = null; if (RepositoryType == WellKnownRepositoryTypes.TfsGit) { // When repository type is TfsGit, set OnPremTfsGit to True, since that variable is added around TFS 2015 Qu2. // Old TFS AT will not send this variable to build agent, and VSTS will always send it to build agent. onPremTfsGit = true; string onPremTfsGitString; if (endpoint.Data.TryGetValue(WellKnownEndpointData.OnPremTfsGit, out onPremTfsGitString)) { onPremTfsGit = StringUtil.ConvertToBoolean(onPremTfsGitString); } if (!_selfManageGitCreds && onPremTfsGit.Value) { // When repository is TFS on-prem git, and we own manage git creds, // we require git version 2.9.0 to support adding extra auth header. _gitCommandManager.EnsureGitVersion(minGitVersionSupportAuthHeader, throwOnNotMatch: true); } } // retrieve credential from endpoint. string username = string.Empty; string password = string.Empty; if (!_selfManageGitCreds && endpoint.Authorization != null) { switch (endpoint.Authorization.Scheme) { case EndpointAuthorizationSchemes.OAuth: username = EndpointAuthorizationSchemes.OAuth; if (!endpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.AccessToken, out password)) { password = string.Empty; } break; case EndpointAuthorizationSchemes.UsernamePassword: if (!endpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.Username, out username)) { // leave the username as empty, the username might in the url, like: http://[email protected] username = string.Empty; } if (!endpoint.Authorization.Parameters.TryGetValue(EndpointAuthorizationParameters.Password, out password)) { // we have username, but no password password = string.Empty; } break; default: executionContext.Warning($"Unsupport endpoint authorization schemes: {endpoint.Authorization.Scheme}"); break; } } // Check the current contents of the root folder to see if there is already a repo // If there is a repo, see if it matches the one we are expecting to be there based on the remote fetch url // if the repo is not what we expect, remove the folder if (!await IsRepositoryOriginUrlMatch(executionContext, targetPath, repositoryUrl)) { // Delete source folder IOUtil.DeleteDirectory(targetPath, cancellationToken); } else { // delete the index.lock file left by previous canceled build or any operation casue git.exe crash last time. string lockFile = Path.Combine(targetPath, ".git\\index.lock"); if (File.Exists(lockFile)) { try { File.Delete(lockFile); } catch (Exception ex) { executionContext.Debug($"Unable to delete the index.lock file: {lockFile}"); executionContext.Debug(ex.ToString()); } } // When repo.clean is selected for a git repo, execute git clean -fdx and git reset --hard HEAD on the current repo. // This will help us save the time to reclone the entire repo. // If any git commands exit with non-zero return code or any exception happened during git.exe invoke, fall back to delete the repo folder. if (clean) { Boolean softClean = false; // git clean -fdx // git reset --hard HEAD int exitCode_clean = await _gitCommandManager.GitClean(executionContext, targetPath); if (exitCode_clean != 0) { executionContext.Debug($"'git clean -fdx' failed with exit code {exitCode_clean}, this normally caused by:\n 1) Path too long\n 2) Permission issue\n 3) File in use\nFor futher investigation, manually run 'git clean -fdx' on repo root: {targetPath} after each build."); } else { int exitCode_reset = await _gitCommandManager.GitReset(executionContext, targetPath); if (exitCode_reset != 0) { executionContext.Debug($"'git reset --hard HEAD' failed with exit code {exitCode_reset}\nFor futher investigation, manually run 'git reset --hard HEAD' on repo root: {targetPath} after each build."); } else { softClean = true; } } if (!softClean) { //fall back executionContext.Warning("Unable to run \"git clean -fdx\" and \"git reset --hard HEAD\" successfully, delete source folder instead."); IOUtil.DeleteDirectory(targetPath, cancellationToken); } } } // if the folder is missing, create it if (!Directory.Exists(targetPath)) { Directory.CreateDirectory(targetPath); } // if the folder contains a .git folder, it means the folder contains a git repo that matches the remote url and in a clean state. // we will run git fetch to update the repo. if (!Directory.Exists(Path.Combine(targetPath, ".git"))) { // init git repository int exitCode_init = await _gitCommandManager.GitInit(executionContext, targetPath); if (exitCode_init != 0) { throw new InvalidOperationException($"Unable to use git.exe init repository under {targetPath}, 'git init' failed with exit code: {exitCode_init}"); } int exitCode_addremote = await _gitCommandManager.GitRemoteAdd(executionContext, targetPath, "origin", repositoryUrl.AbsoluteUri); if (exitCode_addremote != 0) { throw new InvalidOperationException($"Unable to use git.exe add remote 'origin', 'git remote add' failed with exit code: {exitCode_addremote}"); } } cancellationToken.ThrowIfCancellationRequested(); executionContext.Progress(0, "Starting fetch..."); // disable git auto gc int exitCode_disableGC = await _gitCommandManager.GitDisableAutoGC(executionContext, targetPath); if (exitCode_disableGC != 0) { executionContext.Warning("Unable turn off git auto garbage collection, git fetch operation may trigger auto garbage collection which will affect the performence of fetching."); } // always remove any possible left extraheader setting from git config. if (await _gitCommandManager.GitConfigExist(executionContext, targetPath, $"http.{repositoryUrl.AbsoluteUri}.extraheader")) { executionContext.Debug("Remove any extraheader setting from git config."); await RemoveExtraHeader(executionContext, targetPath, $"http.{repositoryUrl.AbsoluteUri}.extraheader", string.Empty); } string additionalFetchArgs = string.Empty; if (!_selfManageGitCreds) { if (RepositoryType == WellKnownRepositoryTypes.TfsGit && (onPremTfsGit.Value || _gitCommandManager.EnsureGitVersion(minGitVersionSupportAuthHeader, throwOnNotMatch: false))) { // When repository is TfsGit on-prem git or vsts-git with git version support adding auth header. additionalFetchArgs = $"-c http.extraheader=\"AUTHORIZATION: bearer {password}\""; } else { // Otherwise, inject credential into fetch/push url // inject credential into fetch url executionContext.Debug("Inject credential into git remote url."); Uri urlWithCred = null; urlWithCred = GetCredentialEmbeddedRepoUrl(repositoryUrl, username, password); // inject credential into fetch url executionContext.Debug("Inject credential into git remote fetch url."); int exitCode_seturl = await _gitCommandManager.GitRemoteSetUrl(executionContext, targetPath, "origin", urlWithCred.AbsoluteUri); if (exitCode_seturl != 0) { throw new InvalidOperationException($"Unable to use git.exe inject credential to git remote fetch url, 'git remote set-url' failed with exit code: {exitCode_seturl}"); } // inject credential into push url executionContext.Debug("Inject credential into git remote push url."); exitCode_seturl = await _gitCommandManager.GitRemoteSetPushUrl(executionContext, targetPath, "origin", urlWithCred.AbsoluteUri); if (exitCode_seturl != 0) { throw new InvalidOperationException($"Unable to use git.exe inject credential to git remote push url, 'git remote set-url --push' failed with exit code: {exitCode_seturl}"); } } } // If this is a build for a pull request, then include // the pull request reference as an additional ref. string fetchSpec = IsPullRequest(sourceBranch) ? StringUtil.Format("+{0}:{1}", sourceBranch, GetRemoteRefName(sourceBranch)) : null; int exitCode_fetch = await _gitCommandManager.GitFetch(executionContext, targetPath, "origin", new List <string>() { fetchSpec }, additionalFetchArgs, cancellationToken); if (exitCode_fetch != 0) { throw new InvalidOperationException($"Git fetch failed with exit code: {exitCode_fetch}"); } // Checkout // sourceToBuild is used for checkout // if sourceBranch is a PR branch or sourceVersion is null, make sure branch name is a remote branch. we need checkout to detached head. // (change refs/heads to refs/remotes/origin, refs/pull to refs/remotes/pull, or leava it as it when the branch name doesn't contain refs/...) // if sourceVersion provide, just use that for checkout, since when you checkout a commit, it will end up in detached head. cancellationToken.ThrowIfCancellationRequested(); executionContext.Progress(80, "Starting checkout..."); string sourcesToBuild; if (IsPullRequest(sourceBranch) || string.IsNullOrEmpty(sourceVersion)) { sourcesToBuild = GetRemoteRefName(sourceBranch); } else { sourcesToBuild = sourceVersion; } // Finally, checkout the sourcesToBuild (if we didn't find a valid git object this will throw) int exitCode_checkout = await _gitCommandManager.GitCheckout(executionContext, targetPath, sourcesToBuild, cancellationToken); if (exitCode_checkout != 0) { throw new InvalidOperationException($"Git checkout failed with exit code: {exitCode_checkout}"); } // Submodule update if (checkoutSubmodules) { cancellationToken.ThrowIfCancellationRequested(); executionContext.Progress(90, "Updating submodules..."); int exitCode_submoduleInit = await _gitCommandManager.GitSubmoduleInit(executionContext, targetPath); if (exitCode_submoduleInit != 0) { throw new InvalidOperationException($"Git submodule init failed with exit code: {exitCode_submoduleInit}"); } string additionalSubmoduleUpdateArgs = string.Empty; if (!_selfManageGitCreds && RepositoryType == WellKnownRepositoryTypes.TfsGit && (onPremTfsGit.Value || _gitCommandManager.EnsureGitVersion(minGitVersionSupportAuthHeader, throwOnNotMatch: false))) { string tfsAccountUrl = repositoryUrl.AbsoluteUri.Replace(repositoryUrl.PathAndQuery, string.Empty); additionalSubmoduleUpdateArgs = $"-c http.{tfsAccountUrl}.extraheader=\"AUTHORIZATION: bearer {password}\""; } int exitCode_submoduleUpdate = await _gitCommandManager.GitSubmoduleUpdate(executionContext, targetPath, additionalSubmoduleUpdateArgs, cancellationToken); if (exitCode_submoduleUpdate != 0) { throw new InvalidOperationException($"Git submodule update failed with exit code: {exitCode_submoduleUpdate}"); } } // handle expose creds, related to 'Allow Scripts to Access OAuth Token' option if (!_selfManageGitCreds) { if (RepositoryType == WellKnownRepositoryTypes.TfsGit && (onPremTfsGit.Value || _gitCommandManager.EnsureGitVersion(minGitVersionSupportAuthHeader, throwOnNotMatch: false))) { if (exposeCred) { string configKey = $"http.{repositoryUrl.AbsoluteUri}.extraheader"; string configValue = $"\"AUTHORIZATION: bearer {password}\""; _authHeaderCache[configKey] = configValue.Trim('\"'); int exitCode_config = await _gitCommandManager.GitConfig(executionContext, targetPath, configKey, configValue); if (exitCode_config != 0) { throw new InvalidOperationException($"Git config failed with exit code: {exitCode_config}"); } } } else { if (!exposeCred) { // remove cached credential from origin's fetch/push url. await RemoveCachedCredential(executionContext, targetPath, repositoryUrl, "origin"); } } } }