public async Task GetSourceAsync(
RunnerActionPluginExecutionContext executionContext,
string repositoryPath,
string repoFullName,
string sourceBranch,
string sourceVersion,
bool clean,
string submoduleInput,
int fetchDepth,
bool gitLfsSupport,
string accessToken,
CancellationToken cancellationToken)
{
// Validate args.
ArgUtil.NotNull(executionContext, nameof(executionContext));
Dictionary <string, string> configModifications = new Dictionary <string, string>();
executionContext.Output($"Syncing repository: {repoFullName}");
Uri repositoryUrl = new Uri($"https://github.com/{repoFullName}");
if (!repositoryUrl.IsAbsoluteUri)
{
throw new InvalidOperationException("Repository url need to be an absolute uri.");
}
string targetPath = repositoryPath;
// input Submodules can be ['', true, false, recursive]
// '' or false indicate don't checkout submodules
// true indicate checkout top level submodules
// recursive indicate checkout submodules recursively
bool checkoutSubmodules = false;
bool checkoutNestedSubmodules = false;
if (!string.IsNullOrEmpty(submoduleInput))
{
if (string.Equals(submoduleInput, Pipelines.PipelineConstants.CheckoutTaskInputs.SubmodulesOptions.Recursive, StringComparison.OrdinalIgnoreCase))
{
checkoutSubmodules = true;
checkoutNestedSubmodules = true;
}
else
{
checkoutSubmodules = StringUtil.ConvertToBoolean(submoduleInput);
}
}
executionContext.Debug($"repository url={repositoryUrl}");
executionContext.Debug($"targetPath={targetPath}");
executionContext.Debug($"sourceBranch={sourceBranch}");
executionContext.Debug($"sourceVersion={sourceVersion}");
executionContext.Debug($"clean={clean}");
executionContext.Debug($"checkoutSubmodules={checkoutSubmodules}");
executionContext.Debug($"checkoutNestedSubmodules={checkoutNestedSubmodules}");
executionContext.Debug($"fetchDepth={fetchDepth}");
executionContext.Debug($"gitLfsSupport={gitLfsSupport}");
// Initialize git command manager with additional environment variables.
Dictionary <string, string> gitEnv = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
// Disable git prompt
gitEnv["GIT_TERMINAL_PROMPT"] = "0";
// Disable prompting for git credential manager
gitEnv["GCM_INTERACTIVE"] = "Never";
// Git-lfs will try to pull down asset if any of the local/user/system setting exist.
// If customer didn't enable `LFS` in their pipeline definition, we will use ENV to disable LFS fetch/checkout.
if (!gitLfsSupport)
{
gitEnv["GIT_LFS_SKIP_SMUDGE"] = "1";
}
// Add the public variables.
foreach (var variable in executionContext.Variables)
{
// Add the variable using the formatted name.
string formattedKey = (variable.Key ?? string.Empty).Replace('.', '_').Replace(' ', '_').ToUpperInvariant();
gitEnv[formattedKey] = variable.Value?.Value ?? string.Empty;
}
GitCliManager gitCommandManager = new GitCliManager(gitEnv);
await gitCommandManager.LoadGitExecutionInfo(executionContext);
// Make sure the build machine met all requirements for the git repository
// For now, the requirement we have are:
// 1. git version greater than 2.9 since we need to use auth header.
// 2. git-lfs version greater than 2.1 since we need to use auth header.
// 3. git version greater than 2.14.2 if use SChannel for SSL backend (Windows only)
RequirementCheck(executionContext, gitCommandManager, gitLfsSupport);
// prepare askpass for client cert private key, if the repository's endpoint url match the runner config url
var systemConnection = executionContext.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
// Check the current contents of the root folder to see if there is already a repo
// If there is a repo, see if it matches the one we are expecting to be there based on the remote fetch url
// if the repo is not what we expect, remove the folder
if (!await IsRepositoryOriginUrlMatch(executionContext, gitCommandManager, targetPath, repositoryUrl))
{
// Delete source folder
IOUtil.DeleteDirectory(targetPath, cancellationToken);
}
else
{
// delete the index.lock file left by previous canceled build or any operation cause git.exe crash last time.
string lockFile = Path.Combine(targetPath, ".git\\index.lock");
if (File.Exists(lockFile))
{
try
{
File.Delete(lockFile);
}
catch (Exception ex)
{
executionContext.Debug($"Unable to delete the index.lock file: {lockFile}");
executionContext.Debug(ex.ToString());
}
}
// delete the shallow.lock file left by previous canceled build or any operation cause git.exe crash last time.
string shallowLockFile = Path.Combine(targetPath, ".git\\shallow.lock");
if (File.Exists(shallowLockFile))
{
try
{
File.Delete(shallowLockFile);
}
catch (Exception ex)
{
executionContext.Debug($"Unable to delete the shallow.lock file: {shallowLockFile}");
executionContext.Debug(ex.ToString());
}
}
// When repo.clean is selected for a git repo, execute git clean -ffdx and git reset --hard HEAD on the current repo.
// This will help us save the time to reclone the entire repo.
// If any git commands exit with non-zero return code or any exception happened during git.exe invoke, fall back to delete the repo folder.
if (clean)
{
Boolean softCleanSucceed = true;
// git clean -ffdx
int exitCode_clean = await gitCommandManager.GitClean(executionContext, targetPath);
if (exitCode_clean != 0)
{
executionContext.Debug($"'git clean -ffdx' failed with exit code {exitCode_clean}, this normally caused by:\n 1) Path too long\n 2) Permission issue\n 3) File in use\nFor futher investigation, manually run 'git clean -ffdx' on repo root: {targetPath} after each build.");
softCleanSucceed = false;
}
// git reset --hard HEAD
if (softCleanSucceed)
{
int exitCode_reset = await gitCommandManager.GitReset(executionContext, targetPath);
if (exitCode_reset != 0)
{
executionContext.Debug($"'git reset --hard HEAD' failed with exit code {exitCode_reset}\nFor futher investigation, manually run 'git reset --hard HEAD' on repo root: {targetPath} after each build.");
softCleanSucceed = false;
}
}
// git clean -ffdx and git reset --hard HEAD for each submodule
if (checkoutSubmodules)
{
if (softCleanSucceed)
{
int exitCode_submoduleclean = await gitCommandManager.GitSubmoduleClean(executionContext, targetPath);
if (exitCode_submoduleclean != 0)
{
executionContext.Debug($"'git submodule foreach git clean -ffdx' failed with exit code {exitCode_submoduleclean}\nFor futher investigation, manually run 'git submodule foreach git clean -ffdx' on repo root: {targetPath} after each build.");
softCleanSucceed = false;
}
}
if (softCleanSucceed)
{
int exitCode_submodulereset = await gitCommandManager.GitSubmoduleReset(executionContext, targetPath);
if (exitCode_submodulereset != 0)
{
executionContext.Debug($"'git submodule foreach git reset --hard HEAD' failed with exit code {exitCode_submodulereset}\nFor futher investigation, manually run 'git submodule foreach git reset --hard HEAD' on repo root: {targetPath} after each build.");
softCleanSucceed = false;
}
}
}
if (!softCleanSucceed)
{
//fall back
executionContext.Warning("Unable to run \"git clean -ffdx\" and \"git reset --hard HEAD\" successfully, delete source folder instead.");
IOUtil.DeleteDirectory(targetPath, cancellationToken);
}
}
}
// if the folder is missing, create it
if (!Directory.Exists(targetPath))
{
Directory.CreateDirectory(targetPath);
}
// if the folder contains a .git folder, it means the folder contains a git repo that matches the remote url and in a clean state.
// we will run git fetch to update the repo.
if (!Directory.Exists(Path.Combine(targetPath, ".git")))
{
// init git repository
int exitCode_init = await gitCommandManager.GitInit(executionContext, targetPath);
if (exitCode_init != 0)
{
throw new InvalidOperationException($"Unable to use git.exe init repository under {targetPath}, 'git init' failed with exit code: {exitCode_init}");
}
int exitCode_addremote = await gitCommandManager.GitRemoteAdd(executionContext, targetPath, "origin", repositoryUrl.AbsoluteUri);
if (exitCode_addremote != 0)
{
throw new InvalidOperationException($"Unable to use git.exe add remote 'origin', 'git remote add' failed with exit code: {exitCode_addremote}");
}
}
cancellationToken.ThrowIfCancellationRequested();
// disable git auto gc
int exitCode_disableGC = await gitCommandManager.GitDisableAutoGC(executionContext, targetPath);
if (exitCode_disableGC != 0)
{
executionContext.Warning("Unable turn off git auto garbage collection, git fetch operation may trigger auto garbage collection which will affect the performance of fetching.");
}
// always remove any possible left extraheader setting from git config.
if (await gitCommandManager.GitConfigExist(executionContext, targetPath, $"http.{repositoryUrl.AbsoluteUri}.extraheader"))
{
executionContext.Debug("Remove any extraheader setting from git config.");
await RemoveGitConfig(executionContext, gitCommandManager, targetPath, $"http.{repositoryUrl.AbsoluteUri}.extraheader", string.Empty);
}
List <string> additionalFetchArgs = new List <string>();
List <string> additionalLfsFetchArgs = new List <string>();
// Add http.https://github.com.extraheader=... to gitconfig
// accessToken as basic auth header to handle any auth challenge from github.com
string configKey = $"http.https://github.com/.extraheader";
string configValue = $"\"AUTHORIZATION: {GenerateBasicAuthHeader(executionContext, accessToken)}\"";
configModifications[configKey] = configValue.Trim('\"');
int exitCode_config = await gitCommandManager.GitConfig(executionContext, targetPath, configKey, configValue);
if (exitCode_config != 0)
{
throw new InvalidOperationException($"Git config failed with exit code: {exitCode_config}");
}
// Prepare gitlfs url for fetch and checkout
if (gitLfsSupport)
{
// Initialize git lfs by execute 'git lfs install'
executionContext.Debug("Setup the local Git hooks for Git LFS.");
int exitCode_lfsInstall = await gitCommandManager.GitLFSInstall(executionContext, targetPath);
if (exitCode_lfsInstall != 0)
{
throw new InvalidOperationException($"Git-lfs installation failed with exit code: {exitCode_lfsInstall}");
}
}
List <string> additionalFetchSpecs = new List <string>();
additionalFetchSpecs.Add("+refs/heads/*:refs/remotes/origin/*");
if (IsPullRequest(sourceBranch))
{
additionalFetchSpecs.Add($"+{sourceBranch}:{GetRemoteRefName(sourceBranch)}");
}
int exitCode_fetch = await gitCommandManager.GitFetch(executionContext, targetPath, "origin", fetchDepth, additionalFetchSpecs, string.Join(" ", additionalFetchArgs), cancellationToken);
if (exitCode_fetch != 0)
{
throw new InvalidOperationException($"Git fetch failed with exit code: {exitCode_fetch}");
}
// Checkout
// sourceToBuild is used for checkout
// if sourceBranch is a PR branch or sourceVersion is null, make sure branch name is a remote branch. we need checkout to detached head.
// (change refs/heads to refs/remotes/origin, refs/pull to refs/remotes/pull, or leave it as it when the branch name doesn't contain refs/...)
// if sourceVersion provide, just use that for checkout, since when you checkout a commit, it will end up in detached head.
cancellationToken.ThrowIfCancellationRequested();
string sourcesToBuild;
if (IsPullRequest(sourceBranch) || string.IsNullOrEmpty(sourceVersion))
{
sourcesToBuild = GetRemoteRefName(sourceBranch);
}
else
{
sourcesToBuild = sourceVersion;
}
// fetch lfs object upfront, this will avoid fetch lfs object during checkout which cause checkout taking forever
// since checkout will fetch lfs object 1 at a time, while git lfs fetch will fetch lfs object in parallel.
if (gitLfsSupport)
{
int exitCode_lfsFetch = await gitCommandManager.GitLFSFetch(executionContext, targetPath, "origin", sourcesToBuild, string.Join(" ", additionalLfsFetchArgs), cancellationToken);
if (exitCode_lfsFetch != 0)
{
// local repository is shallow repository, lfs fetch may fail due to lack of commits history.
// this will happen when the checkout commit is older than tip -> fetchDepth
if (fetchDepth > 0)
{
executionContext.Warning($"Git lfs fetch failed on shallow repository, this might because of git fetch with depth '{fetchDepth}' doesn't include the lfs fetch commit '{sourcesToBuild}'.");
}
// git lfs fetch failed, get lfs log, the log is critical for debug.
int exitCode_lfsLogs = await gitCommandManager.GitLFSLogs(executionContext, targetPath);
throw new InvalidOperationException($"Git lfs fetch failed with exit code: {exitCode_lfsFetch}. Git lfs logs returned with exit code: {exitCode_lfsLogs}.");
}
}
// Finally, checkout the sourcesToBuild (if we didn't find a valid git object this will throw)
int exitCode_checkout = await gitCommandManager.GitCheckout(executionContext, targetPath, sourcesToBuild, cancellationToken);
if (exitCode_checkout != 0)
{
// local repository is shallow repository, checkout may fail due to lack of commits history.
// this will happen when the checkout commit is older than tip -> fetchDepth
if (fetchDepth > 0)
{
executionContext.Warning($"Git checkout failed on shallow repository, this might because of git fetch with depth '{fetchDepth}' doesn't include the checkout commit '{sourcesToBuild}'.");
}
throw new InvalidOperationException($"Git checkout failed with exit code: {exitCode_checkout}");
}
// Submodule update
if (checkoutSubmodules)
{
cancellationToken.ThrowIfCancellationRequested();
int exitCode_submoduleSync = await gitCommandManager.GitSubmoduleSync(executionContext, targetPath, checkoutNestedSubmodules, cancellationToken);
if (exitCode_submoduleSync != 0)
{
throw new InvalidOperationException($"Git submodule sync failed with exit code: {exitCode_submoduleSync}");
}
List <string> additionalSubmoduleUpdateArgs = new List <string>();
int exitCode_submoduleUpdate = await gitCommandManager.GitSubmoduleUpdate(executionContext, targetPath, fetchDepth, string.Join(" ", additionalSubmoduleUpdateArgs), checkoutNestedSubmodules, cancellationToken);
if (exitCode_submoduleUpdate != 0)
{
throw new InvalidOperationException($"Git submodule update failed with exit code: {exitCode_submoduleUpdate}");
}
}
// Set intra-task variable for post job cleanup
executionContext.SetIntraActionState("repositoryPath", targetPath);
executionContext.SetIntraActionState("modifiedgitconfig", JsonUtility.ToString(configModifications.Keys));
foreach (var config in configModifications)
{
executionContext.SetIntraActionState(config.Key, config.Value);
}
}