// git config gc.auto 0 public async Task <int> GitDisableAutoGC(RunnerActionPluginExecutionContext context, string repositoryPath) { context.Debug("Disable git auto garbage collection."); return(await ExecuteGitCommandAsync(context, repositoryPath, "config", "gc.auto 0")); }
// git repack -adfl public async Task <int> GitRepack(RunnerActionPluginExecutionContext context, string repositoryPath) { context.Debug("Compress .git directory."); return(await ExecuteGitCommandAsync(context, repositoryPath, "repack", "-adfl")); }
// git config <key> <value> public async Task <int> GitConfig(RunnerActionPluginExecutionContext context, string repositoryPath, string configKey, string configValue) { context.Debug($"Set git config {configKey} {configValue}"); return(await ExecuteGitCommandAsync(context, repositoryPath, "config", StringUtil.Format($"{configKey} {configValue}"))); }
// git config --unset-all <key> public async Task <int> GitConfigUnset(RunnerActionPluginExecutionContext context, string repositoryPath, string configKey) { context.Debug($"Unset git config --unset-all {configKey}"); return(await ExecuteGitCommandAsync(context, repositoryPath, "config", StringUtil.Format($"--unset-all {configKey}"))); }
private async Task <DownloadResult> DownloadAsync(RunnerActionPluginExecutionContext context, int downloaderId, CancellationToken token) { List <DownloadInfo> failedFiles = new List <DownloadInfo>(); Stopwatch downloadTimer = new Stopwatch(); while (_fileDownloadQueue.TryDequeue(out DownloadInfo fileToDownload)) { token.ThrowIfCancellationRequested(); try { int retryCount = 0; bool downloadFailed = false; while (true) { try { context.Debug($"Start downloading file: '{fileToDownload.ItemPath}' (Downloader {downloaderId})"); downloadTimer.Restart(); using (FileStream fs = new FileStream(fileToDownload.LocalPath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true)) using (var downloadStream = await _fileContainerHttpClient.DownloadFileAsync(_containerId, fileToDownload.ItemPath, token, _projectId)) { await downloadStream.CopyToAsync(fs, _defaultCopyBufferSize, token); await fs.FlushAsync(token); downloadTimer.Stop(); context.Debug($"File: '{fileToDownload.LocalPath}' took {downloadTimer.ElapsedMilliseconds} milliseconds to finish download (Downloader {downloaderId})"); break; } } catch (OperationCanceledException) when(token.IsCancellationRequested) { context.Debug($"Download has been cancelled while downloading {fileToDownload.ItemPath}. (Downloader {downloaderId})"); throw; } catch (Exception ex) { retryCount++; context.Warning($"Fail to download '{fileToDownload.ItemPath}', error: {ex.Message} (Downloader {downloaderId})"); context.Debug(ex.ToString()); } if (retryCount < 3) { var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30)); context.Warning($"Back off {backOff.TotalSeconds} seconds before retry. (Downloader {downloaderId})"); await Task.Delay(backOff); } else { // upload still failed after 3 tries. downloadFailed = true; break; } } if (downloadFailed) { // tracking file that failed to download. failedFiles.Add(fileToDownload); } Interlocked.Increment(ref _downloadFilesProcessed); } catch (Exception ex) { // We should never context.Error($"Error '{ex.Message}' when downloading file '{fileToDownload}'. (Downloader {downloaderId})"); throw; } } return(new DownloadResult(failedFiles)); }
// git submodule foreach git reset --hard HEAD public async Task <int> GitSubmoduleReset(RunnerActionPluginExecutionContext context, string repositoryPath) { context.Debug($"Undo any changes to tracked files in the working tree for submodules at {repositoryPath}."); return(await ExecuteGitCommandAsync(context, repositoryPath, "submodule", "foreach git reset --hard HEAD")); }
// git reset --hard <commit> public async Task <int> GitReset(RunnerActionPluginExecutionContext context, string repositoryPath, string commit = "HEAD") { context.Debug($"Undo any changes to tracked files in the working tree for repository at {repositoryPath}."); return(await ExecuteGitCommandAsync(context, repositoryPath, "reset", $"--hard {commit}")); }
// git lfs install --local public async Task <int> GitLFSInstall(RunnerActionPluginExecutionContext context, string repositoryPath) { context.Debug("Ensure git-lfs installed."); return(await ExecuteGitCommandAsync(context, repositoryPath, "lfs", "install --local")); }
public async Task RunPluginActionAsync(IExecutionContext context, string plugin, Dictionary <string, string> inputs, Dictionary <string, string> environment, Variables runtimeVariables, EventHandler <ProcessDataReceivedEventArgs> outputHandler) { ArgUtil.NotNullOrEmpty(plugin, nameof(plugin)); // Only allow plugins we defined if (!_actionPlugins.Any(x => x.Value.PluginTypeName == plugin || x.Value.PostPluginTypeName == plugin)) { throw new NotSupportedException(plugin); } // Resolve the working directory. string workingDirectory = HostContext.GetDirectory(WellKnownDirectory.Work); ArgUtil.Directory(workingDirectory, nameof(workingDirectory)); // Runner.PluginHost string file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), $"Runner.PluginHost{IOUtil.ExeExtension}"); ArgUtil.File(file, $"Runner.PluginHost{IOUtil.ExeExtension}"); // Runner.PluginHost's arguments string arguments = $"action \"{plugin}\""; // construct plugin context RunnerActionPluginExecutionContext pluginContext = new RunnerActionPluginExecutionContext { Inputs = inputs, Endpoints = context.Endpoints, Context = context.ExpressionValues }; // variables foreach (var variable in context.Variables.AllVariables) { pluginContext.Variables[variable.Name] = new VariableValue(variable.Value, variable.Secret); } using (var processInvoker = HostContext.CreateService <IProcessInvoker>()) { var redirectStandardIn = Channel.CreateUnbounded <string>(new UnboundedChannelOptions() { SingleReader = true, SingleWriter = true }); redirectStandardIn.Writer.TryWrite(JsonUtility.ToString(pluginContext)); processInvoker.OutputDataReceived += outputHandler; processInvoker.ErrorDataReceived += outputHandler; // Execute the process. Exit code 0 should always be returned. // A non-zero exit code indicates infrastructural failure. // Task failure should be communicated over STDOUT using ## commands. await processInvoker.ExecuteAsync(workingDirectory : workingDirectory, fileName : file, arguments : arguments, environment : environment, requireExitCodeZero : true, outputEncoding : Encoding.UTF8, killProcessOnCancel : false, redirectStandardIn : redirectStandardIn, cancellationToken : context.CancellationToken); } }
// git symbolic-ref -q <HEAD> public async Task <int> GitSymbolicRefHEAD(RunnerActionPluginExecutionContext context, string repositoryPath) { context.Debug($"Check whether HEAD is detached HEAD."); return(await ExecuteGitCommandAsync(context, repositoryPath, "symbolic-ref", "-q HEAD")); }
public async Task RunAsync(RunnerActionPluginExecutionContext executionContext, CancellationToken token) { string runnerWorkspace = executionContext.GetRunnerContext("workspace"); ArgUtil.Directory(runnerWorkspace, nameof(runnerWorkspace)); string tempDirectory = executionContext.GetRunnerContext("temp"); ArgUtil.Directory(tempDirectory, nameof(tempDirectory)); var repoFullName = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Repository); if (string.IsNullOrEmpty(repoFullName)) { repoFullName = executionContext.GetGitHubContext("repository"); } var repoFullNameSplit = repoFullName.Split("/", StringSplitOptions.RemoveEmptyEntries); if (repoFullNameSplit.Length != 2) { throw new ArgumentOutOfRangeException(repoFullName); } string expectRepoPath; var path = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Path); if (!string.IsNullOrEmpty(path)) { expectRepoPath = IOUtil.ResolvePath(runnerWorkspace, path); if (!expectRepoPath.StartsWith(runnerWorkspace.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar)) { throw new ArgumentException($"Input path '{path}' should resolve to a directory under '{runnerWorkspace}', current resolved path '{expectRepoPath}'."); } } else { // When repository doesn't has path set, default to sources directory 1/repoName expectRepoPath = Path.Combine(runnerWorkspace, repoFullNameSplit[1]); } var workspaceRepo = executionContext.GetGitHubContext("repository"); // for self repository, we need to let the worker knows where it is after checkout. if (string.Equals(workspaceRepo, repoFullName, StringComparison.OrdinalIgnoreCase)) { var workspaceRepoPath = executionContext.GetGitHubContext("workspace"); executionContext.Debug($"Repository requires to be placed at '{expectRepoPath}', current location is '{workspaceRepoPath}'"); if (!string.Equals(workspaceRepoPath.Trim(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), expectRepoPath.Trim(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), IOUtil.FilePathStringComparison)) { executionContext.Output($"Repository is current at '{workspaceRepoPath}', move to '{expectRepoPath}'."); var count = 1; var staging = Path.Combine(tempDirectory, $"_{count}"); while (Directory.Exists(staging)) { count++; staging = Path.Combine(tempDirectory, $"_{count}"); } try { executionContext.Debug($"Move existing repository '{workspaceRepoPath}' to '{expectRepoPath}' via staging directory '{staging}'."); IOUtil.MoveDirectory(workspaceRepoPath, expectRepoPath, staging, CancellationToken.None); } catch (Exception ex) { executionContext.Debug("Catch exception during repository move."); executionContext.Debug(ex.ToString()); executionContext.Warning("Unable move and reuse existing repository to required location."); IOUtil.DeleteDirectory(expectRepoPath, CancellationToken.None); } executionContext.Output($"Repository will locate at '{expectRepoPath}'."); } executionContext.Debug($"Update workspace repository location."); executionContext.SetRepositoryPath(repoFullName, expectRepoPath, true); } string sourceBranch; string sourceVersion; string refInput = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Ref); if (string.IsNullOrEmpty(refInput)) { sourceBranch = executionContext.GetGitHubContext("ref"); sourceVersion = executionContext.GetGitHubContext("sha"); } else { sourceBranch = refInput; sourceVersion = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Version); // version get removed when checkout move to repo in the graph if (string.IsNullOrEmpty(sourceVersion) && RegexUtility.IsMatch(sourceBranch, WellKnownRegularExpressions.SHA1)) { sourceVersion = sourceBranch; // If Ref is a SHA and the repo is self, we need to use github.ref as source branch since it might be refs/pull/* if (string.Equals(workspaceRepo, repoFullName, StringComparison.OrdinalIgnoreCase)) { sourceBranch = executionContext.GetGitHubContext("ref"); } else { sourceBranch = "refs/heads/master"; } } } bool clean = StringUtil.ConvertToBoolean(executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Clean), true); string submoduleInput = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Submodules); int fetchDepth = 0; if (!int.TryParse(executionContext.GetInput("fetch-depth"), out fetchDepth) || fetchDepth < 0) { fetchDepth = 0; } bool gitLfsSupport = StringUtil.ConvertToBoolean(executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Lfs)); string accessToken = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Token); if (string.IsNullOrEmpty(accessToken)) { accessToken = executionContext.GetGitHubContext("token"); } // register problem matcher string matcherFile = Path.Combine(tempDirectory, $"git_{Guid.NewGuid()}.json"); File.WriteAllText(matcherFile, GitHubSourceProvider.ProblemMatcher, new UTF8Encoding(false)); executionContext.Output($"##[add-matcher]{matcherFile}"); try { await new GitHubSourceProvider().GetSourceAsync(executionContext, expectRepoPath, repoFullName, sourceBranch, sourceVersion, clean, submoduleInput, fetchDepth, gitLfsSupport, accessToken, token); } finally { executionContext.Output("##[remove-matcher owner=checkout-git]"); } }
public async Task DownloadFromContainerAsync( RunnerActionPluginExecutionContext context, String destination, CancellationToken cancellationToken) { // Find out all container items need to be processed List <FileContainerItem> containerItems = new List <FileContainerItem>(); int retryCount = 0; while (retryCount < 3) { try { containerItems = await _fileContainerHttpClient.QueryContainerItemsAsync(_containerId, _projectId, _containerPath, cancellationToken : cancellationToken); break; } catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested) { context.Debug($"Container query has been cancelled."); throw; } catch (Exception ex) when(retryCount < 2) { retryCount++; context.Warning($"Fail to query container items under #/{_containerId}/{_containerPath}, Error: {ex.Message}"); context.Debug(ex.ToString()); } var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15)); context.Warning($"Back off {backOff.TotalSeconds} seconds before retry."); await Task.Delay(backOff); } if (containerItems.Count == 0) { context.Output($"There is nothing under #/{_containerId}/{_containerPath}"); return; } // container items will include both folders, files and even file with zero size // Create all required empty folders and emptry files, gather a list of files that we need to download from server. int foldersCreated = 0; int emptryFilesCreated = 0; List <DownloadInfo> downloadFiles = new List <DownloadInfo>(); foreach (var item in containerItems.OrderBy(x => x.Path)) { if (!item.Path.StartsWith(_containerPath, StringComparison.OrdinalIgnoreCase)) { throw new ArgumentOutOfRangeException($"Item {item.Path} is not under #/{_containerId}/{_containerPath}"); } var localRelativePath = item.Path.Substring(_containerPath.Length).TrimStart('/'); var localPath = Path.Combine(destination, localRelativePath); if (item.ItemType == ContainerItemType.Folder) { context.Debug($"Ensure folder exists: {localPath}"); Directory.CreateDirectory(localPath); foldersCreated++; } else if (item.ItemType == ContainerItemType.File) { if (item.FileLength == 0) { context.Debug($"Create empty file at: {localPath}"); var parentDirectory = Path.GetDirectoryName(localPath); Directory.CreateDirectory(parentDirectory); IOUtil.DeleteFile(localPath); using (new FileStream(localPath, FileMode.Create)) { } emptryFilesCreated++; } else { context.Debug($"Prepare download {item.Path} to {localPath}"); downloadFiles.Add(new DownloadInfo(item.Path, localPath)); } } else { throw new NotSupportedException(item.ItemType.ToString()); } } if (foldersCreated > 0) { context.Output($"{foldersCreated} folders created."); } if (emptryFilesCreated > 0) { context.Output($"{emptryFilesCreated} empty files created."); } if (downloadFiles.Count == 0) { context.Output($"There is nothing to download"); return; } // Start multi-task to download all files. using (_downloadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) { // try download all files for the first time. DownloadResult downloadResult = await ParallelDownloadAsync(context, downloadFiles.AsReadOnly(), Math.Min(downloadFiles.Count, Environment.ProcessorCount), _downloadCancellationTokenSource.Token); if (downloadResult.FailedFiles.Count == 0) { // all files have been download succeed. context.Output($"{downloadFiles.Count} files download succeed."); return; } else { context.Output($"{downloadResult.FailedFiles.Count} files failed to download, retry these files after a minute."); } // Delay 1 min then retry failed files. for (int timer = 60; timer > 0; timer -= 5) { context.Output($"Retry file download after {timer} seconds."); await Task.Delay(TimeSpan.FromSeconds(5), _uploadCancellationTokenSource.Token); } // Retry download all failed files. context.Output($"Start retry {downloadResult.FailedFiles.Count} failed files upload."); DownloadResult retryDownloadResult = await ParallelDownloadAsync(context, downloadResult.FailedFiles.AsReadOnly(), Math.Min(downloadResult.FailedFiles.Count, Environment.ProcessorCount), _downloadCancellationTokenSource.Token); if (retryDownloadResult.FailedFiles.Count == 0) { // all files have been download succeed after retry. context.Output($"{downloadResult.FailedFiles} files download succeed after retry."); return; } else { throw new Exception($"{retryDownloadResult.FailedFiles.Count} files failed to download even after retry."); } } }
private async Task <UploadResult> UploadAsync(RunnerActionPluginExecutionContext context, int uploaderId, CancellationToken token) { List <string> failedFiles = new List <string>(); long uploadedSize = 0; string fileToUpload; Stopwatch uploadTimer = new Stopwatch(); while (_fileUploadQueue.TryDequeue(out fileToUpload)) { token.ThrowIfCancellationRequested(); try { using (FileStream fs = File.Open(fileToUpload, FileMode.Open, FileAccess.Read, FileShare.Read)) { string itemPath = (_containerPath.TrimEnd('/') + "/" + fileToUpload.Remove(0, _sourceParentDirectory.Length + 1)).Replace('\\', '/'); bool failAndExit = false; try { uploadTimer.Restart(); using (HttpResponseMessage response = await _fileContainerHttpClient.UploadFileAsync(_containerId, itemPath, fs, _projectId, cancellationToken: token, chunkSize: 4 * 1024 * 1024)) { if (response == null || response.StatusCode != HttpStatusCode.Created) { context.Output($"Unable to copy file to server StatusCode={response?.StatusCode}: {response?.ReasonPhrase}. Source file path: {fileToUpload}. Target server path: {itemPath}"); if (response?.StatusCode == HttpStatusCode.Conflict) { // fail upload task but continue with any other files context.Error($"Error '{fileToUpload}' has already been uploaded."); } else if (_fileContainerHttpClient.IsFastFailResponse(response)) { // Fast fail: we received an http status code where we should abandon our efforts context.Output($"Cannot continue uploading files, so draining upload queue of {_fileUploadQueue.Count} items."); DrainUploadQueue(context); failedFiles.Clear(); failAndExit = true; throw new UploadFailedException($"Critical failure uploading '{fileToUpload}'"); } else { context.Debug($"Adding '{fileToUpload}' to retry list."); failedFiles.Add(fileToUpload); } throw new UploadFailedException($"Http failure response '{response?.StatusCode}': '{response?.ReasonPhrase}' while uploading '{fileToUpload}'"); } uploadTimer.Stop(); context.Debug($"File: '{fileToUpload}' took {uploadTimer.ElapsedMilliseconds} milliseconds to finish upload"); uploadedSize += fs.Length; OutputLogForFile(context, fileToUpload, $"Detail upload trace for file: {itemPath}", context.Debug); } } catch (OperationCanceledException) when(token.IsCancellationRequested) { context.Output($"File upload has been cancelled during upload file: '{fileToUpload}'."); throw; } catch (Exception ex) { context.Output($"Fail to upload '{fileToUpload}' due to '{ex.Message}'."); context.Output(ex.ToString()); OutputLogForFile(context, fileToUpload, $"Detail upload trace for file that fail to upload: {itemPath}", context.Output); if (failAndExit) { context.Debug("Exiting upload."); throw; } } } Interlocked.Increment(ref _uploadFilesProcessed); } catch (Exception ex) { context.Output($"File error '{ex.Message}' when uploading file '{fileToUpload}'."); throw; } } return(new UploadResult(failedFiles, uploadedSize)); }
// git prune public async Task <int> GitPrune(RunnerActionPluginExecutionContext context, string repositoryPath) { context.Debug("Delete unreachable objects under .git directory."); return(await ExecuteGitCommandAsync(context, repositoryPath, "prune", "-v")); }
// get remote set-url <origin> <url> public async Task <int> GitRemoteAdd(RunnerActionPluginExecutionContext context, string repositoryPath, string remoteName, string remoteUrl) { context.Debug($"Add git remote: {remoteName} to url: {remoteUrl} for repository under: {repositoryPath}."); return(await ExecuteGitCommandAsync(context, repositoryPath, "remote", StringUtil.Format($"add {remoteName} {remoteUrl}"))); }
// git count-objects -v -H public async Task <int> GitCountObjects(RunnerActionPluginExecutionContext context, string repositoryPath) { context.Debug("Inspect .git directory."); return(await ExecuteGitCommandAsync(context, repositoryPath, "count-objects", "-v -H")); }
// get remote set-url --push <origin> <url> public async Task <int> GitRemoteSetPushUrl(RunnerActionPluginExecutionContext context, string repositoryPath, string remoteName, string remoteUrl) { context.Debug($"Set git push url to: {remoteUrl} for remote: {remoteName}."); return(await ExecuteGitCommandAsync(context, repositoryPath, "remote", StringUtil.Format($"set-url --push {remoteName} {remoteUrl}"))); }
// git lfs logs last public async Task <int> GitLFSLogs(RunnerActionPluginExecutionContext context, string repositoryPath) { context.Debug("Get git-lfs logs."); return(await ExecuteGitCommandAsync(context, repositoryPath, "lfs", "logs last")); }
public async Task <long> CopyToContainerAsync( RunnerActionPluginExecutionContext context, String source, CancellationToken cancellationToken) { //set maxConcurrentUploads up to 2 until figure out how to use WinHttpHandler.MaxConnectionsPerServer modify DefaultConnectionLimit int maxConcurrentUploads = Math.Min(Environment.ProcessorCount, 2); //context.Output($"Max Concurrent Uploads {maxConcurrentUploads}"); List <String> files; if (File.Exists(source)) { files = new List <String>() { source }; _sourceParentDirectory = Path.GetDirectoryName(source); } else { files = Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories).ToList(); _sourceParentDirectory = source.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); } context.Output($"Uploading {files.Count()} files"); using (_uploadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) { // hook up reporting event from file container client. _fileContainerHttpClient.UploadFileReportTrace += UploadFileTraceReportReceived; _fileContainerHttpClient.UploadFileReportProgress += UploadFileProgressReportReceived; try { // try upload all files for the first time. UploadResult uploadResult = await ParallelUploadAsync(context, files, maxConcurrentUploads, _uploadCancellationTokenSource.Token); if (uploadResult.RetryFiles.Count == 0) { // all files have been upload succeed. context.Output("File upload complete."); return(uploadResult.TotalFileSizeUploaded); } else { context.Output($"{uploadResult.RetryFiles.Count} files failed to upload, retry these files after a minute."); } // Delay 1 min then retry failed files. for (int timer = 60; timer > 0; timer -= 5) { context.Output($"Retry file upload after {timer} seconds."); await Task.Delay(TimeSpan.FromSeconds(5), _uploadCancellationTokenSource.Token); } // Retry upload all failed files. context.Output($"Start retry {uploadResult.RetryFiles.Count} failed files upload."); UploadResult retryUploadResult = await ParallelUploadAsync(context, uploadResult.RetryFiles, maxConcurrentUploads, _uploadCancellationTokenSource.Token); if (retryUploadResult.RetryFiles.Count == 0) { // all files have been upload succeed after retry. context.Output("File upload complete after retry."); return(uploadResult.TotalFileSizeUploaded + retryUploadResult.TotalFileSizeUploaded); } else { throw new Exception("File upload failed even after retry."); } } finally { _fileContainerHttpClient.UploadFileReportTrace -= UploadFileTraceReportReceived; _fileContainerHttpClient.UploadFileReportProgress -= UploadFileProgressReportReceived; } } }