private async Task RemoveGitConfig(RunnerActionPluginExecutionContext executionContext, GitCliManager gitCommandManager, string targetPath, string configKey, string configValue)
{
int exitCode_configUnset = await gitCommandManager.GitConfigUnset(executionContext, targetPath, configKey);
if (exitCode_configUnset != 0)
{
// if unable to use git.exe unset http.extraheader or core.askpass, modify git config file on disk. make sure we don't left credential.
if (!string.IsNullOrEmpty(configValue))
{
executionContext.Warning("An unsuccessful attempt was made using git command line to remove \"http.extraheader\" from the git config. Attempting to modify the git config file directly to remove the credential.");
string gitConfig = Path.Combine(targetPath, ".git/config");
if (File.Exists(gitConfig))
{
List <string> safeGitConfig = new List <string>();
var gitConfigContents = File.ReadAllLines(gitConfig);
foreach (var line in gitConfigContents)
{
if (!line.Contains(configValue))
{
safeGitConfig.Add(line);
}
}
File.WriteAllLines(gitConfig, safeGitConfig);
}
}
else
{
executionContext.Warning($"Unable to remove \"{configKey}\" from the git config. To remove the credential, execute \"git config --unset - all {configKey}\" from the repository root \"{targetPath}\".");
}
}
}
private async Task RemoveGitConfig(RunnerActionPluginExecutionContext executionContext, GitCliManager gitCommandManager, string targetPath, string configKey, string configValue)
{
int exitCode_configUnset = await gitCommandManager.GitConfigUnset(executionContext, targetPath, configKey);
if (exitCode_configUnset != 0)
{
// if unable to use git.exe unset http.extraheader or core.askpass, modify git config file on disk. make sure we don't left credential.
if (!string.IsNullOrEmpty(configValue))
{
executionContext.Warning("An unsuccessful attempt was made using git command line to remove \"http.extraheader\" from the git config. Attempting to modify the git config file directly to remove the credential.");
string gitConfig = Path.Combine(targetPath, ".git/config");
if (File.Exists(gitConfig))
{
string gitConfigContent = File.ReadAllText(Path.Combine(targetPath, ".git", "config"));
if (gitConfigContent.Contains(configKey))
{
string setting = $"extraheader = {configValue}";
gitConfigContent = Regex.Replace(gitConfigContent, setting, string.Empty, RegexOptions.IgnoreCase);
setting = $"askpass = {configValue}";
gitConfigContent = Regex.Replace(gitConfigContent, setting, string.Empty, RegexOptions.IgnoreCase);
File.WriteAllText(gitConfig, gitConfigContent);
}
}
}
else
{
executionContext.Warning($"Unable to remove \"{configKey}\" from the git config. To remove the credential, execute \"git config --unset - all {configKey}\" from the repository root \"{targetPath}\".");
}
}
}
// git lfs pull
public async Task <int> GitLFSPull(RunnerActionPluginExecutionContext context, string repositoryPath, string additionalCommandLine, CancellationToken cancellationToken)
{
context.Debug($"Download LFS objects for git repository at: {repositoryPath}.");
int retryCount = 0;
int pullExitCode = 0;
while (retryCount < 3)
{
pullExitCode = await ExecuteGitCommandAsync(context, repositoryPath, "lfs", "pull", additionalCommandLine, cancellationToken);
if (pullExitCode == 0)
{
break;
}
else
{
if (++retryCount < 3)
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
context.Warning($"Git lfs pull failed with exit code {pullExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
}
return(pullExitCode);
}
// git lfs fetch origin [ref]
public async Task <int> GitLFSFetch(RunnerActionPluginExecutionContext context, string repositoryPath, string remoteName, string refSpec, string additionalCommandLine, CancellationToken cancellationToken)
{
context.Debug($"Fetch LFS objects for git repository at: {repositoryPath} remote: {remoteName}.");
// default options for git lfs fetch.
string options = StringUtil.Format($"fetch origin {refSpec}");
int retryCount = 0;
int fetchExitCode = 0;
while (retryCount < 3)
{
fetchExitCode = await ExecuteGitCommandAsync(context, repositoryPath, "lfs", options, additionalCommandLine, cancellationToken);
if (fetchExitCode == 0)
{
break;
}
else
{
if (++retryCount < 3)
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
context.Warning($"Git lfs fetch failed with exit code {fetchExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
}
return(fetchExitCode);
}
// git fetch --no-tags --prune --progress --no-recurse-submodules [--depth=15] origin [+refs/pull/*:refs/remote/pull/*] [+refs/tags/1:refs/tags/1]
public async Task <int> GitFetchNoTags(RunnerActionPluginExecutionContext context, string repositoryPath, string remoteName, int fetchDepth, List <string> refSpec, string additionalCommandLine, CancellationToken cancellationToken)
{
context.Debug($"Fetch git repository at: {repositoryPath} remote: {remoteName}.");
if (refSpec != null && refSpec.Count > 0)
{
refSpec = refSpec.Where(r => !string.IsNullOrEmpty(r)).ToList();
}
string options;
// If shallow fetch add --depth arg
// If the local repository is shallowed but there is no fetch depth provide for this build,
// add --unshallow to convert the shallow repository to a complete repository
if (fetchDepth > 0)
{
options = StringUtil.Format($"--no-tags --prune --progress --no-recurse-submodules --depth={fetchDepth} {remoteName} {string.Join(" ", refSpec)}");
}
else if (File.Exists(Path.Combine(repositoryPath, ".git", "shallow")))
{
options = StringUtil.Format($"--no-tags --prune --progress --no-recurse-submodules --unshallow {remoteName} {string.Join(" ", refSpec)}");
}
else
{
// default options for git fetch.
options = StringUtil.Format($"--no-tags --prune --progress --no-recurse-submodules {remoteName} {string.Join(" ", refSpec)}");
}
int retryCount = 0;
int fetchExitCode = 0;
while (retryCount < 3)
{
fetchExitCode = await ExecuteGitCommandAsync(context, repositoryPath, "fetch", options, additionalCommandLine, cancellationToken);
if (fetchExitCode == 0)
{
break;
}
else
{
if (++retryCount < 3)
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
context.Warning($"Git fetch failed with exit code {fetchExitCode}, back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
}
}
return(fetchExitCode);
}
// git config --get remote.origin.url
public async Task <Uri> GitGetFetchUrl(RunnerActionPluginExecutionContext context, string repositoryPath)
{
context.Debug($"Inspect remote.origin.url for repository under {repositoryPath}");
Uri fetchUrl = null;
List <string> outputStrings = new List <string>();
int exitCode = await ExecuteGitCommandAsync(context, repositoryPath, "config", "--get remote.origin.url", outputStrings);
if (exitCode != 0)
{
context.Warning($"'git config --get remote.origin.url' failed with exit code: {exitCode}, output: '{string.Join(Environment.NewLine, outputStrings)}'");
}
else
{
// remove empty strings
outputStrings = outputStrings.Where(o => !string.IsNullOrEmpty(o)).ToList();
if (outputStrings.Count == 1 && !string.IsNullOrEmpty(outputStrings.First()))
{
string remoteFetchUrl = outputStrings.First();
if (Uri.IsWellFormedUriString(remoteFetchUrl, UriKind.Absolute))
{
context.Debug($"Get remote origin fetch url from git config: {remoteFetchUrl}");
fetchUrl = new Uri(remoteFetchUrl);
}
else
{
context.Debug($"The Origin fetch url from git config: {remoteFetchUrl} is not a absolute well formed url.");
}
}
else
{
context.Debug($"Unable capture git remote fetch uri from 'git config --get remote.origin.url' command's output, the command's output is not expected: {string.Join(Environment.NewLine, outputStrings)}.");
}
}
return(fetchUrl);
}
public async Task GetSourceAsync(
RunnerActionPluginExecutionContext executionContext,
string repositoryPath,
string repoFullName,
string sourceBranch,
string sourceVersion,
bool clean,
string submoduleInput,
int fetchDepth,
bool gitLfsSupport,
string accessToken,
CancellationToken cancellationToken)
{
// Validate args.
ArgUtil.NotNull(executionContext, nameof(executionContext));
executionContext.Output($"Syncing repository: {repoFullName}");
Uri repositoryUrl = new Uri($"https://github.com/{repoFullName}");
if (!repositoryUrl.IsAbsoluteUri)
{
throw new InvalidOperationException("Repository url need to be an absolute uri.");
}
string targetPath = repositoryPath;
// input Submodules can be ['', true, false, recursive]
// '' or false indicate don't checkout submodules
// true indicate checkout top level submodules
// recursive indicate checkout submodules recursively
bool checkoutSubmodules = false;
bool checkoutNestedSubmodules = false;
if (!string.IsNullOrEmpty(submoduleInput))
{
if (string.Equals(submoduleInput, Pipelines.PipelineConstants.CheckoutTaskInputs.SubmodulesOptions.Recursive, StringComparison.OrdinalIgnoreCase))
{
checkoutSubmodules = true;
checkoutNestedSubmodules = true;
}
else
{
checkoutSubmodules = StringUtil.ConvertToBoolean(submoduleInput);
}
}
executionContext.Debug($"repository url={repositoryUrl}");
executionContext.Debug($"targetPath={targetPath}");
executionContext.Debug($"sourceBranch={sourceBranch}");
executionContext.Debug($"sourceVersion={sourceVersion}");
executionContext.Debug($"clean={clean}");
executionContext.Debug($"checkoutSubmodules={checkoutSubmodules}");
executionContext.Debug($"checkoutNestedSubmodules={checkoutNestedSubmodules}");
executionContext.Debug($"fetchDepth={fetchDepth}");
executionContext.Debug($"gitLfsSupport={gitLfsSupport}");
// Initialize git command manager with additional environment variables.
Dictionary <string, string> gitEnv = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
// Disable prompting for git credential manager
gitEnv["GCM_INTERACTIVE"] = "Never";
// Git-lfs will try to pull down asset if any of the local/user/system setting exist.
// If customer didn't enable `LFS` in their pipeline definition, we will use ENV to disable LFS fetch/checkout.
if (!gitLfsSupport)
{
gitEnv["GIT_LFS_SKIP_SMUDGE"] = "1";
}
// Add the public variables.
foreach (var variable in executionContext.Variables)
{
// Add the variable using the formatted name.
string formattedKey = (variable.Key ?? string.Empty).Replace('.', '_').Replace(' ', '_').ToUpperInvariant();
gitEnv[formattedKey] = variable.Value?.Value ?? string.Empty;
}
GitCliManager gitCommandManager = new GitCliManager(gitEnv);
await gitCommandManager.LoadGitExecutionInfo(executionContext);
// Make sure the build machine met all requirements for the git repository
// For now, the requirement we have are:
// 1. git version greater than 2.9 since we need to use auth header.
// 2. git-lfs version greater than 2.1 since we need to use auth header.
// 3. git version greater than 2.14.2 if use SChannel for SSL backend (Windows only)
RequirementCheck(executionContext, gitCommandManager, gitLfsSupport);
// prepare askpass for client cert private key, if the repository's endpoint url match the runner config url
var systemConnection = executionContext.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
// Check the current contents of the root folder to see if there is already a repo
// If there is a repo, see if it matches the one we are expecting to be there based on the remote fetch url
// if the repo is not what we expect, remove the folder
if (!await IsRepositoryOriginUrlMatch(executionContext, gitCommandManager, targetPath, repositoryUrl))
{
// Delete source folder
IOUtil.DeleteDirectory(targetPath, cancellationToken);
}
else
{
// delete the index.lock file left by previous canceled build or any operation cause git.exe crash last time.
string lockFile = Path.Combine(targetPath, ".git\\index.lock");
if (File.Exists(lockFile))
{
try
{
File.Delete(lockFile);
}
catch (Exception ex)
{
executionContext.Debug($"Unable to delete the index.lock file: {lockFile}");
executionContext.Debug(ex.ToString());
}
}
// delete the shallow.lock file left by previous canceled build or any operation cause git.exe crash last time.
string shallowLockFile = Path.Combine(targetPath, ".git\\shallow.lock");
if (File.Exists(shallowLockFile))
{
try
{
File.Delete(shallowLockFile);
}
catch (Exception ex)
{
executionContext.Debug($"Unable to delete the shallow.lock file: {shallowLockFile}");
executionContext.Debug(ex.ToString());
}
}
// When repo.clean is selected for a git repo, execute git clean -ffdx and git reset --hard HEAD on the current repo.
// This will help us save the time to reclone the entire repo.
// If any git commands exit with non-zero return code or any exception happened during git.exe invoke, fall back to delete the repo folder.
if (clean)
{
Boolean softCleanSucceed = true;
// git clean -ffdx
int exitCode_clean = await gitCommandManager.GitClean(executionContext, targetPath);
if (exitCode_clean != 0)
{
executionContext.Debug($"'git clean -ffdx' failed with exit code {exitCode_clean}, this normally caused by:\n 1) Path too long\n 2) Permission issue\n 3) File in use\nFor futher investigation, manually run 'git clean -ffdx' on repo root: {targetPath} after each build.");
softCleanSucceed = false;
}
// git reset --hard HEAD
if (softCleanSucceed)
{
int exitCode_reset = await gitCommandManager.GitReset(executionContext, targetPath);
if (exitCode_reset != 0)
{
executionContext.Debug($"'git reset --hard HEAD' failed with exit code {exitCode_reset}\nFor futher investigation, manually run 'git reset --hard HEAD' on repo root: {targetPath} after each build.");
softCleanSucceed = false;
}
}
// git clean -ffdx and git reset --hard HEAD for each submodule
if (checkoutSubmodules)
{
if (softCleanSucceed)
{
int exitCode_submoduleclean = await gitCommandManager.GitSubmoduleClean(executionContext, targetPath);
if (exitCode_submoduleclean != 0)
{
executionContext.Debug($"'git submodule foreach git clean -ffdx' failed with exit code {exitCode_submoduleclean}\nFor futher investigation, manually run 'git submodule foreach git clean -ffdx' on repo root: {targetPath} after each build.");
softCleanSucceed = false;
}
}
if (softCleanSucceed)
{
int exitCode_submodulereset = await gitCommandManager.GitSubmoduleReset(executionContext, targetPath);
if (exitCode_submodulereset != 0)
{
executionContext.Debug($"'git submodule foreach git reset --hard HEAD' failed with exit code {exitCode_submodulereset}\nFor futher investigation, manually run 'git submodule foreach git reset --hard HEAD' on repo root: {targetPath} after each build.");
softCleanSucceed = false;
}
}
}
if (!softCleanSucceed)
{
//fall back
executionContext.Warning("Unable to run \"git clean -ffdx\" and \"git reset --hard HEAD\" successfully, delete source folder instead.");
IOUtil.DeleteDirectory(targetPath, cancellationToken);
}
}
}
// if the folder is missing, create it
if (!Directory.Exists(targetPath))
{
Directory.CreateDirectory(targetPath);
}
// if the folder contains a .git folder, it means the folder contains a git repo that matches the remote url and in a clean state.
// we will run git fetch to update the repo.
if (!Directory.Exists(Path.Combine(targetPath, ".git")))
{
// init git repository
int exitCode_init = await gitCommandManager.GitInit(executionContext, targetPath);
if (exitCode_init != 0)
{
throw new InvalidOperationException($"Unable to use git.exe init repository under {targetPath}, 'git init' failed with exit code: {exitCode_init}");
}
int exitCode_addremote = await gitCommandManager.GitRemoteAdd(executionContext, targetPath, "origin", repositoryUrl.AbsoluteUri);
if (exitCode_addremote != 0)
{
throw new InvalidOperationException($"Unable to use git.exe add remote 'origin', 'git remote add' failed with exit code: {exitCode_addremote}");
}
}
cancellationToken.ThrowIfCancellationRequested();
// disable git auto gc
int exitCode_disableGC = await gitCommandManager.GitDisableAutoGC(executionContext, targetPath);
if (exitCode_disableGC != 0)
{
executionContext.Warning("Unable turn off git auto garbage collection, git fetch operation may trigger auto garbage collection which will affect the performance of fetching.");
}
// always remove any possible left extraheader setting from git config.
if (await gitCommandManager.GitConfigExist(executionContext, targetPath, $"http.{repositoryUrl.AbsoluteUri}.extraheader"))
{
executionContext.Debug("Remove any extraheader setting from git config.");
await RemoveGitConfig(executionContext, gitCommandManager, targetPath, $"http.{repositoryUrl.AbsoluteUri}.extraheader", string.Empty);
}
List <string> additionalFetchArgs = new List <string>();
List <string> additionalLfsFetchArgs = new List <string>();
// add accessToken as basic auth header to handle auth challenge.
if (!string.IsNullOrEmpty(accessToken))
{
additionalFetchArgs.Add($"-c http.extraheader=\"AUTHORIZATION: {GenerateBasicAuthHeader(executionContext, accessToken)}\"");
}
// Prepare gitlfs url for fetch and checkout
if (gitLfsSupport)
{
// Initialize git lfs by execute 'git lfs install'
executionContext.Debug("Setup the local Git hooks for Git LFS.");
int exitCode_lfsInstall = await gitCommandManager.GitLFSInstall(executionContext, targetPath);
if (exitCode_lfsInstall != 0)
{
throw new InvalidOperationException($"Git-lfs installation failed with exit code: {exitCode_lfsInstall}");
}
if (!string.IsNullOrEmpty(accessToken))
{
string authorityUrl = repositoryUrl.AbsoluteUri.Replace(repositoryUrl.PathAndQuery, string.Empty);
additionalLfsFetchArgs.Add($"-c http.{authorityUrl}.extraheader=\"AUTHORIZATION: {GenerateBasicAuthHeader(executionContext, accessToken)}\"");
}
}
List <string> additionalFetchSpecs = new List <string>();
additionalFetchSpecs.Add("+refs/heads/*:refs/remotes/origin/*");
if (IsPullRequest(sourceBranch))
{
additionalFetchSpecs.Add($"+{sourceBranch}:{GetRemoteRefName(sourceBranch)}");
}
int exitCode_fetch = await gitCommandManager.GitFetch(executionContext, targetPath, "origin", fetchDepth, additionalFetchSpecs, string.Join(" ", additionalFetchArgs), cancellationToken);
if (exitCode_fetch != 0)
{
throw new InvalidOperationException($"Git fetch failed with exit code: {exitCode_fetch}");
}
// Checkout
// sourceToBuild is used for checkout
// if sourceBranch is a PR branch or sourceVersion is null, make sure branch name is a remote branch. we need checkout to detached head.
// (change refs/heads to refs/remotes/origin, refs/pull to refs/remotes/pull, or leave it as it when the branch name doesn't contain refs/...)
// if sourceVersion provide, just use that for checkout, since when you checkout a commit, it will end up in detached head.
cancellationToken.ThrowIfCancellationRequested();
string sourcesToBuild;
if (IsPullRequest(sourceBranch) || string.IsNullOrEmpty(sourceVersion))
{
sourcesToBuild = GetRemoteRefName(sourceBranch);
}
else
{
sourcesToBuild = sourceVersion;
}
// fetch lfs object upfront, this will avoid fetch lfs object during checkout which cause checkout taking forever
// since checkout will fetch lfs object 1 at a time, while git lfs fetch will fetch lfs object in parallel.
if (gitLfsSupport)
{
int exitCode_lfsFetch = await gitCommandManager.GitLFSFetch(executionContext, targetPath, "origin", sourcesToBuild, string.Join(" ", additionalLfsFetchArgs), cancellationToken);
if (exitCode_lfsFetch != 0)
{
// local repository is shallow repository, lfs fetch may fail due to lack of commits history.
// this will happen when the checkout commit is older than tip -> fetchDepth
if (fetchDepth > 0)
{
executionContext.Warning($"Git lfs fetch failed on shallow repository, this might because of git fetch with depth '{fetchDepth}' doesn't include the lfs fetch commit '{sourcesToBuild}'.");
}
// git lfs fetch failed, get lfs log, the log is critical for debug.
int exitCode_lfsLogs = await gitCommandManager.GitLFSLogs(executionContext, targetPath);
throw new InvalidOperationException($"Git lfs fetch failed with exit code: {exitCode_lfsFetch}. Git lfs logs returned with exit code: {exitCode_lfsLogs}.");
}
}
// Finally, checkout the sourcesToBuild (if we didn't find a valid git object this will throw)
int exitCode_checkout = await gitCommandManager.GitCheckout(executionContext, targetPath, sourcesToBuild, cancellationToken);
if (exitCode_checkout != 0)
{
// local repository is shallow repository, checkout may fail due to lack of commits history.
// this will happen when the checkout commit is older than tip -> fetchDepth
if (fetchDepth > 0)
{
executionContext.Warning($"Git checkout failed on shallow repository, this might because of git fetch with depth '{fetchDepth}' doesn't include the checkout commit '{sourcesToBuild}'.");
}
throw new InvalidOperationException($"Git checkout failed with exit code: {exitCode_checkout}");
}
// Submodule update
if (checkoutSubmodules)
{
cancellationToken.ThrowIfCancellationRequested();
int exitCode_submoduleSync = await gitCommandManager.GitSubmoduleSync(executionContext, targetPath, checkoutNestedSubmodules, cancellationToken);
if (exitCode_submoduleSync != 0)
{
throw new InvalidOperationException($"Git submodule sync failed with exit code: {exitCode_submoduleSync}");
}
List <string> additionalSubmoduleUpdateArgs = new List <string>();
if (!string.IsNullOrEmpty(accessToken))
{
string authorityUrl = repositoryUrl.AbsoluteUri.Replace(repositoryUrl.PathAndQuery, string.Empty);
additionalSubmoduleUpdateArgs.Add($"-c http.{authorityUrl}.extraheader=\"AUTHORIZATION: {GenerateBasicAuthHeader(executionContext, accessToken)}\"");
}
int exitCode_submoduleUpdate = await gitCommandManager.GitSubmoduleUpdate(executionContext, targetPath, fetchDepth, string.Join(" ", additionalSubmoduleUpdateArgs), checkoutNestedSubmodules, cancellationToken);
if (exitCode_submoduleUpdate != 0)
{
throw new InvalidOperationException($"Git submodule update failed with exit code: {exitCode_submoduleUpdate}");
}
}
}
public async Task RunAsync(RunnerActionPluginExecutionContext executionContext, CancellationToken token)
{
string runnerWorkspace = executionContext.GetRunnerContext("workspace");
ArgUtil.Directory(runnerWorkspace, nameof(runnerWorkspace));
string tempDirectory = executionContext.GetRunnerContext("temp");
ArgUtil.Directory(tempDirectory, nameof(tempDirectory));
var repoFullName = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Repository);
if (string.IsNullOrEmpty(repoFullName))
{
repoFullName = executionContext.GetGitHubContext("repository");
}
var repoFullNameSplit = repoFullName.Split("/", StringSplitOptions.RemoveEmptyEntries);
if (repoFullNameSplit.Length != 2)
{
throw new ArgumentOutOfRangeException(repoFullName);
}
string expectRepoPath;
var path = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Path);
if (!string.IsNullOrEmpty(path))
{
expectRepoPath = IOUtil.ResolvePath(runnerWorkspace, path);
if (!expectRepoPath.StartsWith(runnerWorkspace.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar))
{
throw new ArgumentException($"Input path '{path}' should resolve to a directory under '{runnerWorkspace}', current resolved path '{expectRepoPath}'.");
}
}
else
{
// When repository doesn't has path set, default to sources directory 1/repoName
expectRepoPath = Path.Combine(runnerWorkspace, repoFullNameSplit[1]);
}
var workspaceRepo = executionContext.GetGitHubContext("repository");
// for self repository, we need to let the worker knows where it is after checkout.
if (string.Equals(workspaceRepo, repoFullName, StringComparison.OrdinalIgnoreCase))
{
var workspaceRepoPath = executionContext.GetGitHubContext("workspace");
executionContext.Debug($"Repository requires to be placed at '{expectRepoPath}', current location is '{workspaceRepoPath}'");
if (!string.Equals(workspaceRepoPath.Trim(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), expectRepoPath.Trim(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), IOUtil.FilePathStringComparison))
{
executionContext.Output($"Repository is current at '{workspaceRepoPath}', move to '{expectRepoPath}'.");
var count = 1;
var staging = Path.Combine(tempDirectory, $"_{count}");
while (Directory.Exists(staging))
{
count++;
staging = Path.Combine(tempDirectory, $"_{count}");
}
try
{
executionContext.Debug($"Move existing repository '{workspaceRepoPath}' to '{expectRepoPath}' via staging directory '{staging}'.");
IOUtil.MoveDirectory(workspaceRepoPath, expectRepoPath, staging, CancellationToken.None);
}
catch (Exception ex)
{
executionContext.Debug("Catch exception during repository move.");
executionContext.Debug(ex.ToString());
executionContext.Warning("Unable move and reuse existing repository to required location.");
IOUtil.DeleteDirectory(expectRepoPath, CancellationToken.None);
}
executionContext.Output($"Repository will locate at '{expectRepoPath}'.");
}
executionContext.Debug($"Update workspace repository location.");
executionContext.SetRepositoryPath(repoFullName, expectRepoPath, true);
}
string sourceBranch;
string sourceVersion;
string refInput = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Ref);
if (string.IsNullOrEmpty(refInput))
{
sourceBranch = executionContext.GetGitHubContext("ref");
sourceVersion = executionContext.GetGitHubContext("sha");
}
else
{
sourceBranch = refInput;
sourceVersion = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Version); // version get removed when checkout move to repo in the graph
if (string.IsNullOrEmpty(sourceVersion) && RegexUtility.IsMatch(sourceBranch, WellKnownRegularExpressions.SHA1))
{
sourceVersion = sourceBranch;
// If Ref is a SHA and the repo is self, we need to use github.ref as source branch since it might be refs/pull/*
if (string.Equals(workspaceRepo, repoFullName, StringComparison.OrdinalIgnoreCase))
{
sourceBranch = executionContext.GetGitHubContext("ref");
}
else
{
sourceBranch = "refs/heads/master";
}
}
}
bool clean = StringUtil.ConvertToBoolean(executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Clean), true);
string submoduleInput = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Submodules);
int fetchDepth = 0;
if (!int.TryParse(executionContext.GetInput("fetch-depth"), out fetchDepth) || fetchDepth < 0)
{
fetchDepth = 0;
}
bool gitLfsSupport = StringUtil.ConvertToBoolean(executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Lfs));
string accessToken = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Token);
if (string.IsNullOrEmpty(accessToken))
{
accessToken = executionContext.GetGitHubContext("token");
}
// register problem matcher
string problemMatcher = @"
{
""problemMatcher"": [
{
""owner"": ""checkout-git"",
""pattern"": [
{
""regexp"": ""^fatal: (.*)$"",
""message"": 1
}
]
}
]
}";
string matcherFile = Path.Combine(tempDirectory, $"git_{Guid.NewGuid()}.json");
File.WriteAllText(matcherFile, problemMatcher, new UTF8Encoding(false));
executionContext.Output($"##[add-matcher]{matcherFile}");
try
{
await new GitHubSourceProvider().GetSourceAsync(executionContext,
expectRepoPath,
repoFullName,
sourceBranch,
sourceVersion,
clean,
submoduleInput,
fetchDepth,
gitLfsSupport,
accessToken,
token);
}
finally
{
executionContext.Output("##[remove-matcher owner=checkout-git]");
}
}
public async Task DownloadFromContainerAsync(
RunnerActionPluginExecutionContext context,
String destination,
CancellationToken cancellationToken)
{
// Find out all container items need to be processed
List <FileContainerItem> containerItems = new List <FileContainerItem>();
int retryCount = 0;
while (retryCount < 3)
{
try
{
containerItems = await _fileContainerHttpClient.QueryContainerItemsAsync(_containerId,
_projectId,
_containerPath,
cancellationToken : cancellationToken);
break;
}
catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
{
context.Debug($"Container query has been cancelled.");
throw;
}
catch (Exception ex) when(retryCount < 2)
{
retryCount++;
context.Warning($"Fail to query container items under #/{_containerId}/{_containerPath}, Error: {ex.Message}");
context.Debug(ex.ToString());
}
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
context.Warning($"Back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
if (containerItems.Count == 0)
{
context.Output($"There is nothing under #/{_containerId}/{_containerPath}");
return;
}
// container items will include both folders, files and even file with zero size
// Create all required empty folders and emptry files, gather a list of files that we need to download from server.
int foldersCreated = 0;
int emptryFilesCreated = 0;
List <DownloadInfo> downloadFiles = new List <DownloadInfo>();
foreach (var item in containerItems.OrderBy(x => x.Path))
{
if (!item.Path.StartsWith(_containerPath, StringComparison.OrdinalIgnoreCase))
{
throw new ArgumentOutOfRangeException($"Item {item.Path} is not under #/{_containerId}/{_containerPath}");
}
var localRelativePath = item.Path.Substring(_containerPath.Length).TrimStart('/');
var localPath = Path.Combine(destination, localRelativePath);
if (item.ItemType == ContainerItemType.Folder)
{
context.Debug($"Ensure folder exists: {localPath}");
Directory.CreateDirectory(localPath);
foldersCreated++;
}
else if (item.ItemType == ContainerItemType.File)
{
if (item.FileLength == 0)
{
context.Debug($"Create empty file at: {localPath}");
var parentDirectory = Path.GetDirectoryName(localPath);
Directory.CreateDirectory(parentDirectory);
IOUtil.DeleteFile(localPath);
using (new FileStream(localPath, FileMode.Create))
{
}
emptryFilesCreated++;
}
else
{
context.Debug($"Prepare download {item.Path} to {localPath}");
downloadFiles.Add(new DownloadInfo(item.Path, localPath));
}
}
else
{
throw new NotSupportedException(item.ItemType.ToString());
}
}
if (foldersCreated > 0)
{
context.Output($"{foldersCreated} folders created.");
}
if (emptryFilesCreated > 0)
{
context.Output($"{emptryFilesCreated} empty files created.");
}
if (downloadFiles.Count == 0)
{
context.Output($"There is nothing to download");
return;
}
// Start multi-task to download all files.
using (_downloadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
// try download all files for the first time.
DownloadResult downloadResult = await ParallelDownloadAsync(context, downloadFiles.AsReadOnly(), Math.Min(downloadFiles.Count, Environment.ProcessorCount), _downloadCancellationTokenSource.Token);
if (downloadResult.FailedFiles.Count == 0)
{
// all files have been download succeed.
context.Output($"{downloadFiles.Count} files download succeed.");
return;
}
else
{
context.Output($"{downloadResult.FailedFiles.Count} files failed to download, retry these files after a minute.");
}
// Delay 1 min then retry failed files.
for (int timer = 60; timer > 0; timer -= 5)
{
context.Output($"Retry file download after {timer} seconds.");
await Task.Delay(TimeSpan.FromSeconds(5), _uploadCancellationTokenSource.Token);
}
// Retry download all failed files.
context.Output($"Start retry {downloadResult.FailedFiles.Count} failed files upload.");
DownloadResult retryDownloadResult = await ParallelDownloadAsync(context, downloadResult.FailedFiles.AsReadOnly(), Math.Min(downloadResult.FailedFiles.Count, Environment.ProcessorCount), _downloadCancellationTokenSource.Token);
if (retryDownloadResult.FailedFiles.Count == 0)
{
// all files have been download succeed after retry.
context.Output($"{downloadResult.FailedFiles} files download succeed after retry.");
return;
}
else
{
throw new Exception($"{retryDownloadResult.FailedFiles.Count} files failed to download even after retry.");
}
}
}
private async Task <DownloadResult> DownloadAsync(RunnerActionPluginExecutionContext context, int downloaderId, CancellationToken token)
{
List <DownloadInfo> failedFiles = new List <DownloadInfo>();
Stopwatch downloadTimer = new Stopwatch();
while (_fileDownloadQueue.TryDequeue(out DownloadInfo fileToDownload))
{
token.ThrowIfCancellationRequested();
try
{
int retryCount = 0;
bool downloadFailed = false;
while (true)
{
try
{
context.Debug($"Start downloading file: '{fileToDownload.ItemPath}' (Downloader {downloaderId})");
downloadTimer.Restart();
using (FileStream fs = new FileStream(fileToDownload.LocalPath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true))
using (var downloadStream = await _fileContainerHttpClient.DownloadFileAsync(_containerId, fileToDownload.ItemPath, token, _projectId))
{
await downloadStream.CopyToAsync(fs, _defaultCopyBufferSize, token);
await fs.FlushAsync(token);
downloadTimer.Stop();
context.Debug($"File: '{fileToDownload.LocalPath}' took {downloadTimer.ElapsedMilliseconds} milliseconds to finish download (Downloader {downloaderId})");
break;
}
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
context.Debug($"Download has been cancelled while downloading {fileToDownload.ItemPath}. (Downloader {downloaderId})");
throw;
}
catch (Exception ex)
{
retryCount++;
context.Warning($"Fail to download '{fileToDownload.ItemPath}', error: {ex.Message} (Downloader {downloaderId})");
context.Debug(ex.ToString());
}
if (retryCount < 3)
{
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30));
context.Warning($"Back off {backOff.TotalSeconds} seconds before retry. (Downloader {downloaderId})");
await Task.Delay(backOff);
}
else
{
// upload still failed after 3 tries.
downloadFailed = true;
break;
}
}
if (downloadFailed)
{
// tracking file that failed to download.
failedFiles.Add(fileToDownload);
}
Interlocked.Increment(ref _downloadFilesProcessed);
}
catch (Exception ex)
{
// We should never
context.Error($"Error '{ex.Message}' when downloading file '{fileToDownload}'. (Downloader {downloaderId})");
throw;
}
}
return(new DownloadResult(failedFiles));
}