Esempio n. 1
0
 // git config gc.auto 0
 public async Task <int> GitDisableAutoGC(RunnerActionPluginExecutionContext context, string repositoryPath)
 {
     context.Debug("Disable git auto garbage collection.");
     return(await ExecuteGitCommandAsync(context, repositoryPath, "config", "gc.auto 0"));
 }
Esempio n. 2
0
 // git repack -adfl
 public async Task <int> GitRepack(RunnerActionPluginExecutionContext context, string repositoryPath)
 {
     context.Debug("Compress .git directory.");
     return(await ExecuteGitCommandAsync(context, repositoryPath, "repack", "-adfl"));
 }
Esempio n. 3
0
 // git config <key> <value>
 public async Task <int> GitConfig(RunnerActionPluginExecutionContext context, string repositoryPath, string configKey, string configValue)
 {
     context.Debug($"Set git config {configKey} {configValue}");
     return(await ExecuteGitCommandAsync(context, repositoryPath, "config", StringUtil.Format($"{configKey} {configValue}")));
 }
Esempio n. 4
0
 // git config --unset-all <key>
 public async Task <int> GitConfigUnset(RunnerActionPluginExecutionContext context, string repositoryPath, string configKey)
 {
     context.Debug($"Unset git config --unset-all {configKey}");
     return(await ExecuteGitCommandAsync(context, repositoryPath, "config", StringUtil.Format($"--unset-all {configKey}")));
 }
        private async Task <DownloadResult> DownloadAsync(RunnerActionPluginExecutionContext context, int downloaderId, CancellationToken token)
        {
            List <DownloadInfo> failedFiles   = new List <DownloadInfo>();
            Stopwatch           downloadTimer = new Stopwatch();

            while (_fileDownloadQueue.TryDequeue(out DownloadInfo fileToDownload))
            {
                token.ThrowIfCancellationRequested();
                try
                {
                    int  retryCount     = 0;
                    bool downloadFailed = false;
                    while (true)
                    {
                        try
                        {
                            context.Debug($"Start downloading file: '{fileToDownload.ItemPath}' (Downloader {downloaderId})");
                            downloadTimer.Restart();
                            using (FileStream fs = new FileStream(fileToDownload.LocalPath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true))
                                using (var downloadStream = await _fileContainerHttpClient.DownloadFileAsync(_containerId, fileToDownload.ItemPath, token, _projectId))
                                {
                                    await downloadStream.CopyToAsync(fs, _defaultCopyBufferSize, token);

                                    await fs.FlushAsync(token);

                                    downloadTimer.Stop();
                                    context.Debug($"File: '{fileToDownload.LocalPath}' took {downloadTimer.ElapsedMilliseconds} milliseconds to finish download (Downloader {downloaderId})");
                                    break;
                                }
                        }
                        catch (OperationCanceledException) when(token.IsCancellationRequested)
                        {
                            context.Debug($"Download has been cancelled while downloading {fileToDownload.ItemPath}. (Downloader {downloaderId})");
                            throw;
                        }
                        catch (Exception ex)
                        {
                            retryCount++;
                            context.Warning($"Fail to download '{fileToDownload.ItemPath}', error: {ex.Message} (Downloader {downloaderId})");
                            context.Debug(ex.ToString());
                        }

                        if (retryCount < 3)
                        {
                            var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30));
                            context.Warning($"Back off {backOff.TotalSeconds} seconds before retry. (Downloader {downloaderId})");
                            await Task.Delay(backOff);
                        }
                        else
                        {
                            // upload still failed after 3 tries.
                            downloadFailed = true;
                            break;
                        }
                    }

                    if (downloadFailed)
                    {
                        // tracking file that failed to download.
                        failedFiles.Add(fileToDownload);
                    }

                    Interlocked.Increment(ref _downloadFilesProcessed);
                }
                catch (Exception ex)
                {
                    // We should never
                    context.Error($"Error '{ex.Message}' when downloading file '{fileToDownload}'. (Downloader {downloaderId})");
                    throw;
                }
            }

            return(new DownloadResult(failedFiles));
        }
Esempio n. 6
0
 // git submodule foreach git reset --hard HEAD
 public async Task <int> GitSubmoduleReset(RunnerActionPluginExecutionContext context, string repositoryPath)
 {
     context.Debug($"Undo any changes to tracked files in the working tree for submodules at {repositoryPath}.");
     return(await ExecuteGitCommandAsync(context, repositoryPath, "submodule", "foreach git reset --hard HEAD"));
 }
Esempio n. 7
0
 // git reset --hard <commit>
 public async Task <int> GitReset(RunnerActionPluginExecutionContext context, string repositoryPath, string commit = "HEAD")
 {
     context.Debug($"Undo any changes to tracked files in the working tree for repository at {repositoryPath}.");
     return(await ExecuteGitCommandAsync(context, repositoryPath, "reset", $"--hard {commit}"));
 }
Esempio n. 8
0
 // git lfs install --local
 public async Task <int> GitLFSInstall(RunnerActionPluginExecutionContext context, string repositoryPath)
 {
     context.Debug("Ensure git-lfs installed.");
     return(await ExecuteGitCommandAsync(context, repositoryPath, "lfs", "install --local"));
 }
Esempio n. 9
0
        public async Task RunPluginActionAsync(IExecutionContext context, string plugin, Dictionary <string, string> inputs, Dictionary <string, string> environment, Variables runtimeVariables, EventHandler <ProcessDataReceivedEventArgs> outputHandler)
        {
            ArgUtil.NotNullOrEmpty(plugin, nameof(plugin));

            // Only allow plugins we defined
            if (!_actionPlugins.Any(x => x.Value.PluginTypeName == plugin || x.Value.PostPluginTypeName == plugin))
            {
                throw new NotSupportedException(plugin);
            }

            // Resolve the working directory.
            string workingDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);

            ArgUtil.Directory(workingDirectory, nameof(workingDirectory));

            // Runner.PluginHost
            string file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), $"Runner.PluginHost{IOUtil.ExeExtension}");

            ArgUtil.File(file, $"Runner.PluginHost{IOUtil.ExeExtension}");

            // Runner.PluginHost's arguments
            string arguments = $"action \"{plugin}\"";

            // construct plugin context
            RunnerActionPluginExecutionContext pluginContext = new RunnerActionPluginExecutionContext
            {
                Inputs    = inputs,
                Endpoints = context.Endpoints,
                Context   = context.ExpressionValues
            };

            // variables
            foreach (var variable in context.Variables.AllVariables)
            {
                pluginContext.Variables[variable.Name] = new VariableValue(variable.Value, variable.Secret);
            }

            using (var processInvoker = HostContext.CreateService <IProcessInvoker>())
            {
                var redirectStandardIn = Channel.CreateUnbounded <string>(new UnboundedChannelOptions()
                {
                    SingleReader = true, SingleWriter = true
                });
                redirectStandardIn.Writer.TryWrite(JsonUtility.ToString(pluginContext));

                processInvoker.OutputDataReceived += outputHandler;
                processInvoker.ErrorDataReceived  += outputHandler;

                // Execute the process. Exit code 0 should always be returned.
                // A non-zero exit code indicates infrastructural failure.
                // Task failure should be communicated over STDOUT using ## commands.
                await processInvoker.ExecuteAsync(workingDirectory : workingDirectory,
                                                  fileName : file,
                                                  arguments : arguments,
                                                  environment : environment,
                                                  requireExitCodeZero : true,
                                                  outputEncoding : Encoding.UTF8,
                                                  killProcessOnCancel : false,
                                                  redirectStandardIn : redirectStandardIn,
                                                  cancellationToken : context.CancellationToken);
            }
        }
Esempio n. 10
0
 // git symbolic-ref -q <HEAD>
 public async Task <int> GitSymbolicRefHEAD(RunnerActionPluginExecutionContext context, string repositoryPath)
 {
     context.Debug($"Check whether HEAD is detached HEAD.");
     return(await ExecuteGitCommandAsync(context, repositoryPath, "symbolic-ref", "-q HEAD"));
 }
Esempio n. 11
0
        public async Task RunAsync(RunnerActionPluginExecutionContext executionContext, CancellationToken token)
        {
            string runnerWorkspace = executionContext.GetRunnerContext("workspace");

            ArgUtil.Directory(runnerWorkspace, nameof(runnerWorkspace));
            string tempDirectory = executionContext.GetRunnerContext("temp");

            ArgUtil.Directory(tempDirectory, nameof(tempDirectory));

            var repoFullName = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Repository);

            if (string.IsNullOrEmpty(repoFullName))
            {
                repoFullName = executionContext.GetGitHubContext("repository");
            }

            var repoFullNameSplit = repoFullName.Split("/", StringSplitOptions.RemoveEmptyEntries);

            if (repoFullNameSplit.Length != 2)
            {
                throw new ArgumentOutOfRangeException(repoFullName);
            }

            string expectRepoPath;
            var    path = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Path);

            if (!string.IsNullOrEmpty(path))
            {
                expectRepoPath = IOUtil.ResolvePath(runnerWorkspace, path);
                if (!expectRepoPath.StartsWith(runnerWorkspace.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar))
                {
                    throw new ArgumentException($"Input path '{path}' should resolve to a directory under '{runnerWorkspace}', current resolved path '{expectRepoPath}'.");
                }
            }
            else
            {
                // When repository doesn't has path set, default to sources directory 1/repoName
                expectRepoPath = Path.Combine(runnerWorkspace, repoFullNameSplit[1]);
            }

            var workspaceRepo = executionContext.GetGitHubContext("repository");

            // for self repository, we need to let the worker knows where it is after checkout.
            if (string.Equals(workspaceRepo, repoFullName, StringComparison.OrdinalIgnoreCase))
            {
                var workspaceRepoPath = executionContext.GetGitHubContext("workspace");

                executionContext.Debug($"Repository requires to be placed at '{expectRepoPath}', current location is '{workspaceRepoPath}'");
                if (!string.Equals(workspaceRepoPath.Trim(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), expectRepoPath.Trim(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), IOUtil.FilePathStringComparison))
                {
                    executionContext.Output($"Repository is current at '{workspaceRepoPath}', move to '{expectRepoPath}'.");
                    var count   = 1;
                    var staging = Path.Combine(tempDirectory, $"_{count}");
                    while (Directory.Exists(staging))
                    {
                        count++;
                        staging = Path.Combine(tempDirectory, $"_{count}");
                    }

                    try
                    {
                        executionContext.Debug($"Move existing repository '{workspaceRepoPath}' to '{expectRepoPath}' via staging directory '{staging}'.");
                        IOUtil.MoveDirectory(workspaceRepoPath, expectRepoPath, staging, CancellationToken.None);
                    }
                    catch (Exception ex)
                    {
                        executionContext.Debug("Catch exception during repository move.");
                        executionContext.Debug(ex.ToString());
                        executionContext.Warning("Unable move and reuse existing repository to required location.");
                        IOUtil.DeleteDirectory(expectRepoPath, CancellationToken.None);
                    }

                    executionContext.Output($"Repository will locate at '{expectRepoPath}'.");
                }

                executionContext.Debug($"Update workspace repository location.");
                executionContext.SetRepositoryPath(repoFullName, expectRepoPath, true);
            }

            string sourceBranch;
            string sourceVersion;
            string refInput = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Ref);

            if (string.IsNullOrEmpty(refInput))
            {
                sourceBranch  = executionContext.GetGitHubContext("ref");
                sourceVersion = executionContext.GetGitHubContext("sha");
            }
            else
            {
                sourceBranch  = refInput;
                sourceVersion = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Version);  // version get removed when checkout move to repo in the graph
                if (string.IsNullOrEmpty(sourceVersion) && RegexUtility.IsMatch(sourceBranch, WellKnownRegularExpressions.SHA1))
                {
                    sourceVersion = sourceBranch;
                    // If Ref is a SHA and the repo is self, we need to use github.ref as source branch since it might be refs/pull/*
                    if (string.Equals(workspaceRepo, repoFullName, StringComparison.OrdinalIgnoreCase))
                    {
                        sourceBranch = executionContext.GetGitHubContext("ref");
                    }
                    else
                    {
                        sourceBranch = "refs/heads/master";
                    }
                }
            }

            bool   clean          = StringUtil.ConvertToBoolean(executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Clean), true);
            string submoduleInput = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Submodules);

            int fetchDepth = 0;

            if (!int.TryParse(executionContext.GetInput("fetch-depth"), out fetchDepth) || fetchDepth < 0)
            {
                fetchDepth = 0;
            }

            bool   gitLfsSupport = StringUtil.ConvertToBoolean(executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Lfs));
            string accessToken   = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.Token);

            if (string.IsNullOrEmpty(accessToken))
            {
                accessToken = executionContext.GetGitHubContext("token");
            }

            // register problem matcher
            string matcherFile = Path.Combine(tempDirectory, $"git_{Guid.NewGuid()}.json");

            File.WriteAllText(matcherFile, GitHubSourceProvider.ProblemMatcher, new UTF8Encoding(false));
            executionContext.Output($"##[add-matcher]{matcherFile}");
            try
            {
                await new GitHubSourceProvider().GetSourceAsync(executionContext,
                                                                expectRepoPath,
                                                                repoFullName,
                                                                sourceBranch,
                                                                sourceVersion,
                                                                clean,
                                                                submoduleInput,
                                                                fetchDepth,
                                                                gitLfsSupport,
                                                                accessToken,
                                                                token);
            }
            finally
            {
                executionContext.Output("##[remove-matcher owner=checkout-git]");
            }
        }
Esempio n. 12
0
        public async Task DownloadFromContainerAsync(
            RunnerActionPluginExecutionContext context,
            String destination,
            CancellationToken cancellationToken)
        {
            // Find out all container items need to be processed
            List <FileContainerItem> containerItems = new List <FileContainerItem>();
            int retryCount = 0;

            while (retryCount < 3)
            {
                try
                {
                    containerItems = await _fileContainerHttpClient.QueryContainerItemsAsync(_containerId,
                                                                                             _projectId,
                                                                                             _containerPath,
                                                                                             cancellationToken : cancellationToken);

                    break;
                }
                catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
                {
                    context.Debug($"Container query has been cancelled.");
                    throw;
                }
                catch (Exception ex) when(retryCount < 2)
                {
                    retryCount++;
                    context.Warning($"Fail to query container items under #/{_containerId}/{_containerPath}, Error: {ex.Message}");
                    context.Debug(ex.ToString());
                }

                var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
                context.Warning($"Back off {backOff.TotalSeconds} seconds before retry.");
                await Task.Delay(backOff);
            }

            if (containerItems.Count == 0)
            {
                context.Output($"There is nothing under #/{_containerId}/{_containerPath}");
                return;
            }

            // container items will include both folders, files and even file with zero size
            // Create all required empty folders and emptry files, gather a list of files that we need to download from server.
            int foldersCreated                = 0;
            int emptryFilesCreated            = 0;
            List <DownloadInfo> downloadFiles = new List <DownloadInfo>();

            foreach (var item in containerItems.OrderBy(x => x.Path))
            {
                if (!item.Path.StartsWith(_containerPath, StringComparison.OrdinalIgnoreCase))
                {
                    throw new ArgumentOutOfRangeException($"Item {item.Path} is not under #/{_containerId}/{_containerPath}");
                }

                var localRelativePath = item.Path.Substring(_containerPath.Length).TrimStart('/');
                var localPath         = Path.Combine(destination, localRelativePath);

                if (item.ItemType == ContainerItemType.Folder)
                {
                    context.Debug($"Ensure folder exists: {localPath}");
                    Directory.CreateDirectory(localPath);
                    foldersCreated++;
                }
                else if (item.ItemType == ContainerItemType.File)
                {
                    if (item.FileLength == 0)
                    {
                        context.Debug($"Create empty file at: {localPath}");
                        var parentDirectory = Path.GetDirectoryName(localPath);
                        Directory.CreateDirectory(parentDirectory);
                        IOUtil.DeleteFile(localPath);
                        using (new FileStream(localPath, FileMode.Create))
                        {
                        }
                        emptryFilesCreated++;
                    }
                    else
                    {
                        context.Debug($"Prepare download {item.Path} to {localPath}");
                        downloadFiles.Add(new DownloadInfo(item.Path, localPath));
                    }
                }
                else
                {
                    throw new NotSupportedException(item.ItemType.ToString());
                }
            }

            if (foldersCreated > 0)
            {
                context.Output($"{foldersCreated} folders created.");
            }

            if (emptryFilesCreated > 0)
            {
                context.Output($"{emptryFilesCreated} empty files created.");
            }

            if (downloadFiles.Count == 0)
            {
                context.Output($"There is nothing to download");
                return;
            }

            // Start multi-task to download all files.
            using (_downloadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
            {
                // try download all files for the first time.
                DownloadResult downloadResult = await ParallelDownloadAsync(context, downloadFiles.AsReadOnly(), Math.Min(downloadFiles.Count, Environment.ProcessorCount), _downloadCancellationTokenSource.Token);

                if (downloadResult.FailedFiles.Count == 0)
                {
                    // all files have been download succeed.
                    context.Output($"{downloadFiles.Count} files download succeed.");
                    return;
                }
                else
                {
                    context.Output($"{downloadResult.FailedFiles.Count} files failed to download, retry these files after a minute.");
                }

                // Delay 1 min then retry failed files.
                for (int timer = 60; timer > 0; timer -= 5)
                {
                    context.Output($"Retry file download after {timer} seconds.");
                    await Task.Delay(TimeSpan.FromSeconds(5), _uploadCancellationTokenSource.Token);
                }

                // Retry download all failed files.
                context.Output($"Start retry {downloadResult.FailedFiles.Count} failed files upload.");
                DownloadResult retryDownloadResult = await ParallelDownloadAsync(context, downloadResult.FailedFiles.AsReadOnly(), Math.Min(downloadResult.FailedFiles.Count, Environment.ProcessorCount), _downloadCancellationTokenSource.Token);

                if (retryDownloadResult.FailedFiles.Count == 0)
                {
                    // all files have been download succeed after retry.
                    context.Output($"{downloadResult.FailedFiles} files download succeed after retry.");
                    return;
                }
                else
                {
                    throw new Exception($"{retryDownloadResult.FailedFiles.Count} files failed to download even after retry.");
                }
            }
        }
Esempio n. 13
0
        private async Task <UploadResult> UploadAsync(RunnerActionPluginExecutionContext context, int uploaderId, CancellationToken token)
        {
            List <string> failedFiles  = new List <string>();
            long          uploadedSize = 0;
            string        fileToUpload;
            Stopwatch     uploadTimer = new Stopwatch();

            while (_fileUploadQueue.TryDequeue(out fileToUpload))
            {
                token.ThrowIfCancellationRequested();
                try
                {
                    using (FileStream fs = File.Open(fileToUpload, FileMode.Open, FileAccess.Read, FileShare.Read))
                    {
                        string itemPath    = (_containerPath.TrimEnd('/') + "/" + fileToUpload.Remove(0, _sourceParentDirectory.Length + 1)).Replace('\\', '/');
                        bool   failAndExit = false;
                        try
                        {
                            uploadTimer.Restart();
                            using (HttpResponseMessage response = await _fileContainerHttpClient.UploadFileAsync(_containerId, itemPath, fs, _projectId, cancellationToken: token, chunkSize: 4 * 1024 * 1024))
                            {
                                if (response == null || response.StatusCode != HttpStatusCode.Created)
                                {
                                    context.Output($"Unable to copy file to server StatusCode={response?.StatusCode}: {response?.ReasonPhrase}. Source file path: {fileToUpload}. Target server path: {itemPath}");

                                    if (response?.StatusCode == HttpStatusCode.Conflict)
                                    {
                                        // fail upload task but continue with any other files
                                        context.Error($"Error '{fileToUpload}' has already been uploaded.");
                                    }
                                    else if (_fileContainerHttpClient.IsFastFailResponse(response))
                                    {
                                        // Fast fail: we received an http status code where we should abandon our efforts
                                        context.Output($"Cannot continue uploading files, so draining upload queue of {_fileUploadQueue.Count} items.");
                                        DrainUploadQueue(context);
                                        failedFiles.Clear();
                                        failAndExit = true;
                                        throw new UploadFailedException($"Critical failure uploading '{fileToUpload}'");
                                    }
                                    else
                                    {
                                        context.Debug($"Adding '{fileToUpload}' to retry list.");
                                        failedFiles.Add(fileToUpload);
                                    }
                                    throw new UploadFailedException($"Http failure response '{response?.StatusCode}': '{response?.ReasonPhrase}' while uploading '{fileToUpload}'");
                                }

                                uploadTimer.Stop();
                                context.Debug($"File: '{fileToUpload}' took {uploadTimer.ElapsedMilliseconds} milliseconds to finish upload");
                                uploadedSize += fs.Length;
                                OutputLogForFile(context, fileToUpload, $"Detail upload trace for file: {itemPath}", context.Debug);
                            }
                        }
                        catch (OperationCanceledException) when(token.IsCancellationRequested)
                        {
                            context.Output($"File upload has been cancelled during upload file: '{fileToUpload}'.");
                            throw;
                        }
                        catch (Exception ex)
                        {
                            context.Output($"Fail to upload '{fileToUpload}' due to '{ex.Message}'.");
                            context.Output(ex.ToString());

                            OutputLogForFile(context, fileToUpload, $"Detail upload trace for file that fail to upload: {itemPath}", context.Output);

                            if (failAndExit)
                            {
                                context.Debug("Exiting upload.");
                                throw;
                            }
                        }
                    }

                    Interlocked.Increment(ref _uploadFilesProcessed);
                }
                catch (Exception ex)
                {
                    context.Output($"File error '{ex.Message}' when uploading file '{fileToUpload}'.");
                    throw;
                }
            }

            return(new UploadResult(failedFiles, uploadedSize));
        }
Esempio n. 14
0
 // git prune
 public async Task <int> GitPrune(RunnerActionPluginExecutionContext context, string repositoryPath)
 {
     context.Debug("Delete unreachable objects under .git directory.");
     return(await ExecuteGitCommandAsync(context, repositoryPath, "prune", "-v"));
 }
Esempio n. 15
0
 // get remote set-url <origin> <url>
 public async Task <int> GitRemoteAdd(RunnerActionPluginExecutionContext context, string repositoryPath, string remoteName, string remoteUrl)
 {
     context.Debug($"Add git remote: {remoteName} to url: {remoteUrl} for repository under: {repositoryPath}.");
     return(await ExecuteGitCommandAsync(context, repositoryPath, "remote", StringUtil.Format($"add {remoteName} {remoteUrl}")));
 }
Esempio n. 16
0
 // git count-objects -v -H
 public async Task <int> GitCountObjects(RunnerActionPluginExecutionContext context, string repositoryPath)
 {
     context.Debug("Inspect .git directory.");
     return(await ExecuteGitCommandAsync(context, repositoryPath, "count-objects", "-v -H"));
 }
Esempio n. 17
0
 // get remote set-url --push <origin> <url>
 public async Task <int> GitRemoteSetPushUrl(RunnerActionPluginExecutionContext context, string repositoryPath, string remoteName, string remoteUrl)
 {
     context.Debug($"Set git push url to: {remoteUrl} for remote: {remoteName}.");
     return(await ExecuteGitCommandAsync(context, repositoryPath, "remote", StringUtil.Format($"set-url --push {remoteName} {remoteUrl}")));
 }
Esempio n. 18
0
 // git lfs logs last
 public async Task <int> GitLFSLogs(RunnerActionPluginExecutionContext context, string repositoryPath)
 {
     context.Debug("Get git-lfs logs.");
     return(await ExecuteGitCommandAsync(context, repositoryPath, "lfs", "logs last"));
 }
Esempio n. 19
0
        public async Task <long> CopyToContainerAsync(
            RunnerActionPluginExecutionContext context,
            String source,
            CancellationToken cancellationToken)
        {
            //set maxConcurrentUploads up to 2 until figure out how to use WinHttpHandler.MaxConnectionsPerServer modify DefaultConnectionLimit
            int maxConcurrentUploads = Math.Min(Environment.ProcessorCount, 2);
            //context.Output($"Max Concurrent Uploads {maxConcurrentUploads}");

            List <String> files;

            if (File.Exists(source))
            {
                files = new List <String>()
                {
                    source
                };
                _sourceParentDirectory = Path.GetDirectoryName(source);
            }
            else
            {
                files = Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories).ToList();
                _sourceParentDirectory = source.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
            }

            context.Output($"Uploading {files.Count()} files");
            using (_uploadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
            {
                // hook up reporting event from file container client.
                _fileContainerHttpClient.UploadFileReportTrace    += UploadFileTraceReportReceived;
                _fileContainerHttpClient.UploadFileReportProgress += UploadFileProgressReportReceived;

                try
                {
                    // try upload all files for the first time.
                    UploadResult uploadResult = await ParallelUploadAsync(context, files, maxConcurrentUploads, _uploadCancellationTokenSource.Token);

                    if (uploadResult.RetryFiles.Count == 0)
                    {
                        // all files have been upload succeed.
                        context.Output("File upload complete.");
                        return(uploadResult.TotalFileSizeUploaded);
                    }
                    else
                    {
                        context.Output($"{uploadResult.RetryFiles.Count} files failed to upload, retry these files after a minute.");
                    }

                    // Delay 1 min then retry failed files.
                    for (int timer = 60; timer > 0; timer -= 5)
                    {
                        context.Output($"Retry file upload after {timer} seconds.");
                        await Task.Delay(TimeSpan.FromSeconds(5), _uploadCancellationTokenSource.Token);
                    }

                    // Retry upload all failed files.
                    context.Output($"Start retry {uploadResult.RetryFiles.Count} failed files upload.");
                    UploadResult retryUploadResult = await ParallelUploadAsync(context, uploadResult.RetryFiles, maxConcurrentUploads, _uploadCancellationTokenSource.Token);

                    if (retryUploadResult.RetryFiles.Count == 0)
                    {
                        // all files have been upload succeed after retry.
                        context.Output("File upload complete after retry.");
                        return(uploadResult.TotalFileSizeUploaded + retryUploadResult.TotalFileSizeUploaded);
                    }
                    else
                    {
                        throw new Exception("File upload failed even after retry.");
                    }
                }
                finally
                {
                    _fileContainerHttpClient.UploadFileReportTrace    -= UploadFileTraceReportReceived;
                    _fileContainerHttpClient.UploadFileReportProgress -= UploadFileProgressReportReceived;
                }
            }
        }