private bool Match(ServerBuildArtifact buildArtifact, ArtifactDefinition artifactDefinition) { //TODO: If editing artifactDefinitionName is not allowed then we can remove this if (string.Equals(artifactDefinition.Name, AllArtifacts, StringComparison.OrdinalIgnoreCase)) { return(true); } if (string.Equals(artifactDefinition.Name, buildArtifact.Name, StringComparison.OrdinalIgnoreCase)) { return(true); } return(false); }
private static void OutputArtifactDefinition(ArtifactDefinition artifactArtifactDefinition) { Log.Warn(" -Definition: "); Log.Info(" -Description: "+ artifactArtifactDefinition.BusinessDescription); Log.Info(""); Log.Info(" -Example: "+ artifactArtifactDefinition.BusinessExample); Log.Info(""); Log.Warn(" -Analogies: "); foreach (var a in artifactArtifactDefinition.Analogies) { Log.Info(" -Analogy: "+ a.Name); Log.Info(" -Description: "+ a.Description); } Log.Info(" -Comments: "+ artifactArtifactDefinition.Comments); }
public async Task DownloadAsync( IExecutionContext executionContext, ArtifactDefinition artifactDefinition, string localFolderPath) { ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(artifactDefinition, nameof(artifactDefinition)); ArgUtil.NotNullOrEmpty(localFolderPath, nameof(localFolderPath)); var gitHubDetails = artifactDefinition.Details as GitHubArtifactDetails; ArgUtil.NotNull(gitHubDetails, nameof(gitHubDetails)); executionContext.Output(StringUtil.Loc("RMReceivedGithubArtifactDetails")); ServiceEndpoint endpoint = executionContext.Endpoints.FirstOrDefault((e => string.Equals(e.Name, gitHubDetails.ConnectionName, StringComparison.OrdinalIgnoreCase))); if (endpoint == null) { throw new InvalidOperationException(StringUtil.Loc("RMGitHubEndpointNotFound", gitHubDetails.ConnectionName)); } ServiceEndpoint gitHubEndpoint = PrepareGitHubTaskEndpoint(endpoint, gitHubDetails.CloneUrl); var extensionManager = HostContext.GetService <IExtensionManager>(); ISourceProvider sourceProvider = (extensionManager.GetExtensions <ISourceProvider>()).FirstOrDefault(x => x.RepositoryType == Microsoft.TeamFoundation.DistributedTask.Pipelines.RepositoryTypes.GitHub); if (sourceProvider == null) { throw new InvalidOperationException(StringUtil.Loc("SourceArtifactProviderNotFound", Microsoft.TeamFoundation.DistributedTask.Pipelines.RepositoryTypes.GitHub)); } gitHubEndpoint.Data.Add(Constants.EndpointData.SourcesDirectory, localFolderPath); gitHubEndpoint.Data.Add(Constants.EndpointData.SourceBranch, gitHubDetails.Branch); gitHubEndpoint.Data.Add(Constants.EndpointData.SourceVersion, artifactDefinition.Version); gitHubEndpoint.Data.Add(EndpointData.CheckoutSubmodules, gitHubDetails.CheckoutSubmodules); gitHubEndpoint.Data.Add(EndpointData.CheckoutNestedSubmodules, gitHubDetails.CheckoutNestedSubmodules); gitHubEndpoint.Data.Add("fetchDepth", gitHubDetails.FetchDepth); gitHubEndpoint.Data.Add("GitLfsSupport", gitHubDetails.GitLfsSupport); try { await sourceProvider.GetSourceAsync(executionContext, gitHubEndpoint, executionContext.CancellationToken); } catch (InvalidOperationException ex) { throw new ArtifactDownloadException(StringUtil.Loc("RMDownloadArtifactUnexpectedError"), ex); } }
private ArtifactDefinition ConvertToArtifactDefinition(AgentArtifactDefinition agentArtifactDefinition, IExecutionContext executionContext, IArtifactExtension extension) { Trace.Entering(); ArgUtil.NotNull(agentArtifactDefinition, nameof(agentArtifactDefinition)); ArgUtil.NotNull(executionContext, nameof(executionContext)); var artifactDefinition = new ArtifactDefinition { ArtifactType = agentArtifactDefinition.ArtifactType, Name = agentArtifactDefinition.Name, Version = agentArtifactDefinition.Version }; artifactDefinition.Details = extension.GetArtifactDetails(executionContext, agentArtifactDefinition); return(artifactDefinition); }
public async Task DownloadAsync(IExecutionContext executionContext, ArtifactDefinition artifactDefinition, string downloadFolderPath) { ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(artifactDefinition, nameof(artifactDefinition)); ArgUtil.NotNullOrEmpty(downloadFolderPath, nameof(downloadFolderPath)); var gitArtifactDetails = artifactDefinition.Details as TfsGitArtifactDetails; ArgUtil.NotNull(gitArtifactDetails, nameof(gitArtifactDetails)); ServiceEndpoint endpoint = executionContext.Endpoints.FirstOrDefault((e => string.Equals(e.Name, gitArtifactDetails.RepositoryId, StringComparison.OrdinalIgnoreCase))); if (endpoint == null) { throw new InvalidOperationException(StringUtil.Loc("RMGitEndpointNotFound")); } var extensionManager = HostContext.GetService <IExtensionManager>(); ISourceProvider sourceProvider = (extensionManager.GetExtensions <ISourceProvider>()).FirstOrDefault(x => x.RepositoryType == RepositoryTypes.TfsGit); if (sourceProvider == null) { throw new InvalidOperationException(StringUtil.Loc("SourceArtifactProviderNotFound", RepositoryTypes.TfsGit)); } var tfsGitEndpoint = endpoint.Clone(); tfsGitEndpoint.Data.Add(Constants.EndpointData.SourcesDirectory, downloadFolderPath); tfsGitEndpoint.Data.Add(Constants.EndpointData.SourceBranch, gitArtifactDetails.Branch); tfsGitEndpoint.Data.Add(Constants.EndpointData.SourceVersion, artifactDefinition.Version); tfsGitEndpoint.Data.Add(EndpointData.CheckoutSubmodules, gitArtifactDetails.CheckoutSubmodules); tfsGitEndpoint.Data.Add(EndpointData.CheckoutNestedSubmodules, gitArtifactDetails.CheckoutNestedSubmodules); tfsGitEndpoint.Data.Add("fetchDepth", gitArtifactDetails.FetchDepth); tfsGitEndpoint.Data.Add("GitLfsSupport", gitArtifactDetails.GitLfsSupport); try { await sourceProvider.GetSourceAsync(executionContext, tfsGitEndpoint, executionContext.CancellationToken); } catch (InvalidOperationException ex) { throw new ArtifactDownloadException(StringUtil.Loc("RMDownloadArtifactUnexpectedError"), ex); } }
private ArtifactDefinition ConvertToArtifactDefinition(AgentArtifactDefinition agentArtifactDefinition, IExecutionContext executionContext, IArtifactExtension extension) { Trace.Entering(); ArgUtil.NotNull(agentArtifactDefinition, nameof(agentArtifactDefinition)); ArgUtil.NotNull(executionContext, nameof(executionContext)); var artifactDefinition = new ArtifactDefinition { ArtifactType = agentArtifactDefinition.ArtifactType, Name = agentArtifactDefinition.Name, Version = agentArtifactDefinition.Version }; RetryExecutor retryExecutor = new RetryExecutor(); retryExecutor.ShouldRetryAction = (ex) => { bool retry = true; if (ex is InvalidOperationException) { retry = false; } else { Trace.Warning(ex.ToString()); } return(retry); }; retryExecutor.Execute( () => { artifactDefinition.Details = extension.GetArtifactDetails(executionContext, agentArtifactDefinition); }); return(artifactDefinition); }
private TestHostContext Setup([CallerMemberName] string name = "") { TestHostContext hc = new TestHostContext(this, name); _ec = new Mock <IExecutionContext>(); _httpClient = new Mock <IGenericHttpClient>(); _artifactDefinition = new ArtifactDefinition { Details = new JenkinsArtifactDetails { Url = new Uri("http://localhost"), JobName = "jenkins", Alias = "jenkins" } }; _extensionManager = new Mock <IExtensionManager>(); hc.SetSingleton <IExtensionManager>(_extensionManager.Object); hc.SetSingleton <IGenericHttpClient>(_httpClient.Object); return(hc); }
public async Task DownloadAsync(IExecutionContext executionContext, ArtifactDefinition artifactDefinition, string downloadFolderPath) { ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(artifactDefinition, nameof(artifactDefinition)); ArgUtil.NotNullOrEmpty(downloadFolderPath, nameof(downloadFolderPath)); var tfsVcArtifactDetails = artifactDefinition.Details as TfsVCArtifactDetails; ArgUtil.NotNull(tfsVcArtifactDetails, nameof(tfsVcArtifactDetails)); ServiceEndpoint endpoint = executionContext.Endpoints.FirstOrDefault((e => string.Equals(e.Name, tfsVcArtifactDetails.RepositoryId, StringComparison.OrdinalIgnoreCase))); if (endpoint == null) { throw new InvalidOperationException(StringUtil.Loc("RMTfsVCEndpointNotFound")); } var tfsVCEndpoint = endpoint.Clone(); PrepareTfsVCEndpoint(tfsVCEndpoint, tfsVcArtifactDetails); var extensionManager = HostContext.GetService <IExtensionManager>(); ISourceProvider sourceProvider = (extensionManager.GetExtensions <ISourceProvider>()).FirstOrDefault(x => x.RepositoryType == Microsoft.TeamFoundation.DistributedTask.Pipelines.RepositoryTypes.Tfvc); if (sourceProvider == null) { throw new InvalidOperationException(StringUtil.Loc("SourceArtifactProviderNotFound", Microsoft.TeamFoundation.DistributedTask.Pipelines.RepositoryTypes.Tfvc)); } var rootDirectory = Directory.GetParent(downloadFolderPath).Name; executionContext.Variables.Set(Constants.Variables.Agent.BuildDirectory, rootDirectory); tfsVCEndpoint.Data.Add(Constants.EndpointData.SourcesDirectory, downloadFolderPath); tfsVCEndpoint.Data.Add(Constants.EndpointData.SourceVersion, artifactDefinition.Version); await sourceProvider.GetSourceAsync(executionContext, tfsVCEndpoint, executionContext.CancellationToken); }
public async Task DownloadAsync( IExecutionContext executionContext, ArtifactDefinition artifactDefinition, string localFolderPath) { ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNull(artifactDefinition, nameof(artifactDefinition)); ArgUtil.NotNullOrEmpty(localFolderPath, nameof(localFolderPath)); var gitHubDetails = artifactDefinition.Details as GitHubArtifactDetails; ArgUtil.NotNull(gitHubDetails, nameof(gitHubDetails)); executionContext.Output(StringUtil.Loc("RMReceivedGithubArtifactDetails")); ServiceEndpoint endpoint = executionContext.Endpoints.FirstOrDefault((e => string.Equals(e.Name, gitHubDetails.ConnectionName, StringComparison.OrdinalIgnoreCase))); if (endpoint == null) { throw new InvalidOperationException(StringUtil.Loc("RMGitHubEndpointNotFound", gitHubDetails.ConnectionName)); } ServiceEndpoint gitHubEndpoint = PrepareGitHubTaskEndpoint(endpoint, gitHubDetails.CloneUrl); var extensionManager = HostContext.GetService <IExtensionManager>(); ISourceProvider sourceProvider = (extensionManager.GetExtensions <ISourceProvider>()).FirstOrDefault(x => x.RepositoryType == WellKnownRepositoryTypes.GitHub); if (sourceProvider == null) { throw new InvalidOperationException(StringUtil.Loc("SourceArtifactProviderNotFound", WellKnownRepositoryTypes.GitHub)); } gitHubEndpoint.Data.Add(Constants.EndpointData.SourcesDirectory, localFolderPath); gitHubEndpoint.Data.Add(Constants.EndpointData.SourceBranch, gitHubDetails.Branch); gitHubEndpoint.Data.Add(Constants.EndpointData.SourceVersion, artifactDefinition.Version); await sourceProvider.GetSourceAsync(executionContext, gitHubEndpoint, executionContext.CancellationToken); }
private async Task DownloadArtifactAsync( IExecutionContext executionContext, ServerBuildArtifact buildArtifact, ArtifactDefinition artifactDefinition, string localFolderPath) { var downloadFolderPath = Path.Combine(localFolderPath, buildArtifact.Name); var buildArtifactDetails = artifactDefinition.Details as BuildArtifactDetails; if ((buildArtifact.Resource.Type == null && buildArtifact.Id == 0) || // bug on build API Bug 378900 string.Equals(buildArtifact.Resource.Type, ArtifactResourceTypes.FilePath, StringComparison.OrdinalIgnoreCase)) { executionContext.Output(StringUtil.Loc("RMArtifactTypeFileShare")); if (!PlatformUtil.RunningOnWindows) { throw new NotSupportedException(StringUtil.Loc("RMFileShareArtifactErrorOnNonWindowsAgent")); } string fileShare; if (buildArtifact.Id == 0) { fileShare = new Uri(buildArtifact.Resource.DownloadUrl).LocalPath; } else { fileShare = new Uri(Path.Combine(buildArtifact.Resource.DownloadUrl, buildArtifact.Name)).LocalPath; if (!Directory.Exists(fileShare)) { // download path does not exist, log and fall back var parenthPath = new Uri(buildArtifact.Resource.DownloadUrl).LocalPath; executionContext.Output(StringUtil.Loc("RMArtifactNameDirectoryNotFound", fileShare, parenthPath)); fileShare = parenthPath; } } if (!Directory.Exists(fileShare)) { // download path does not exist, raise exception throw new ArtifactDownloadException(StringUtil.Loc("RMArtifactDirectoryNotFoundError", fileShare, WindowsIdentity.GetCurrent().Name)); } executionContext.Output(StringUtil.Loc("RMDownloadingArtifactFromFileShare", fileShare, downloadFolderPath)); var fileShareArtifact = new FileShareArtifact(); await fileShareArtifact.DownloadArtifactAsync(executionContext, HostContext, artifactDefinition, fileShare, downloadFolderPath); } else if (buildArtifactDetails != null && string.Equals(buildArtifact.Resource.Type, ArtifactResourceTypes.Container, StringComparison.OrdinalIgnoreCase)) { executionContext.Output(StringUtil.Loc("RMArtifactTypeServerDrop")); // Get containerId and rootLocation for the artifact #/922702/drop string containerUrl = buildArtifact.Resource.Data; string[] parts = containerUrl.Split(new[] { '/' }, 3); if (parts.Length < 3) { throw new ArtifactDownloadException(StringUtil.Loc("RMArtifactContainerDetailsNotFoundError", buildArtifact.Name)); } int containerId; string rootLocation = parts[2]; if (!int.TryParse(parts[1], out containerId)) { throw new ArtifactDownloadException(StringUtil.Loc("RMArtifactContainerDetailsInvalidError", buildArtifact.Name)); } string rootDestinationDir = Path.Combine(localFolderPath, rootLocation); executionContext.Output(StringUtil.Loc("RMDownloadingArtifactFromFileContainer", containerUrl, rootDestinationDir)); var containerFetchEngineOptions = new ContainerFetchEngineOptions { ParallelDownloadLimit = executionContext.Variables.Release_Parallel_Download_Limit ?? ContainerFetchEngineDefaultOptions.ParallelDownloadLimit, DownloadBufferSize = executionContext.Variables.Release_Download_BufferSize ?? ContainerFetchEngineDefaultOptions.DownloadBufferSize }; executionContext.Output(StringUtil.Loc("RMParallelDownloadLimit", containerFetchEngineOptions.ParallelDownloadLimit)); executionContext.Output(StringUtil.Loc("RMDownloadBufferSize", containerFetchEngineOptions.DownloadBufferSize)); IContainerProvider containerProvider = new ContainerProviderFactory(buildArtifactDetails, rootLocation, containerId, executionContext).GetContainerProvider( ArtifactResourceTypes.Container); using (var engine = new ContainerFetchEngine.ContainerFetchEngine(containerProvider, rootLocation, rootDestinationDir)) { engine.ContainerFetchEngineOptions = containerFetchEngineOptions; engine.ExecutionLogger = new ExecutionLogger(executionContext); await engine.FetchAsync(executionContext.CancellationToken); } } else { executionContext.Warning(StringUtil.Loc("RMArtifactTypeNotSupported", buildArtifact.Resource.Type)); } }
private async Task DownloadArtifacts(IExecutionContext executionContext, List <AgentArtifactDefinition> agentArtifactDefinitions, string artifactsWorkingFolder) { Trace.Entering(); foreach (AgentArtifactDefinition agentArtifactDefinition in agentArtifactDefinitions) { // We don't need to check if its old style artifact anymore. All the build data has been fixed and all the build artifact has Alias now. ArgUtil.NotNullOrEmpty(agentArtifactDefinition.Alias, nameof(agentArtifactDefinition.Alias)); var extensionManager = HostContext.GetService <IExtensionManager>(); IArtifactExtension extension = (extensionManager.GetExtensions <IArtifactExtension>()).FirstOrDefault(x => agentArtifactDefinition.ArtifactType == x.ArtifactType); if (extension == null) { throw new InvalidOperationException(StringUtil.Loc("RMArtifactTypeNotSupported")); } Trace.Info($"Found artifact extension of type {extension.ArtifactType}"); executionContext.Output(StringUtil.Loc("RMStartArtifactsDownload")); ArtifactDefinition artifactDefinition = ConvertToArtifactDefinition(agentArtifactDefinition, executionContext, extension); executionContext.Output(StringUtil.Loc("RMArtifactDownloadBegin", agentArtifactDefinition.Alias)); executionContext.Output(StringUtil.Loc("RMDownloadArtifactType", agentArtifactDefinition.ArtifactType)); // Get the local path where this artifact should be downloaded. string downloadFolderPath = Path.GetFullPath(Path.Combine(artifactsWorkingFolder, agentArtifactDefinition.Alias ?? string.Empty)); // Create the directory if it does not exist. if (!Directory.Exists(downloadFolderPath)) { // TODO: old windows agent has a directory cache, verify and implement it if its required. Directory.CreateDirectory(downloadFolderPath); executionContext.Output(StringUtil.Loc("RMArtifactFolderCreated", downloadFolderPath)); } // download the artifact to this path. RetryExecutor retryExecutor = new RetryExecutor(); retryExecutor.ShouldRetryAction = (ex) => { executionContext.Output(StringUtil.Loc("RMErrorDuringArtifactDownload", ex)); bool retry = true; if (ex is ArtifactDownloadException) { retry = false; } else { executionContext.Output(StringUtil.Loc("RMRetryingArtifactDownload")); } return(retry); }; await retryExecutor.ExecuteAsync( async() => { //TODO:SetAttributesToNormal var releaseFileSystemManager = HostContext.GetService <IReleaseFileSystemManager>(); releaseFileSystemManager.CleanupDirectory(downloadFolderPath, executionContext.CancellationToken); if (agentArtifactDefinition.ArtifactType == AgentArtifactType.GitHub || agentArtifactDefinition.ArtifactType == AgentArtifactType.TFGit || agentArtifactDefinition.ArtifactType == AgentArtifactType.Tfvc) { throw new NotImplementedException(); } else { await extension.DownloadAsync(executionContext, artifactDefinition, downloadFolderPath); } }); executionContext.Output(StringUtil.Loc("RMArtifactDownloadFinished", agentArtifactDefinition.Alias)); } executionContext.Output(StringUtil.Loc("RMArtifactsDownloadFinished")); }
private async Task DownloadArtifactAsync( IExecutionContext executionContext, ServerBuildArtifact buildArtifact, ArtifactDefinition artifactDefinition, string localFolderPath, BuildHttpClient buildClient, XamlBuildHttpClient xamlBuildClient, DefinitionType definitionType, int buildId) { var downloadFolderPath = Path.Combine(localFolderPath, buildArtifact.Name); var buildArtifactDetails = artifactDefinition.Details as BuildArtifactDetails; if ((buildArtifact.Resource.Type == null && buildArtifact.Id == 0) || // bug on build API Bug 378900 string.Equals(buildArtifact.Resource.Type, WellKnownArtifactResourceTypes.FilePath, StringComparison.OrdinalIgnoreCase)) { executionContext.Output("Artifact Type: FileShare"); string fileShare; if (buildArtifact.Id == 0) { fileShare = new Uri(buildArtifact.Resource.DownloadUrl).LocalPath; } else { fileShare = new Uri(Path.Combine(buildArtifact.Resource.DownloadUrl, buildArtifact.Name)).LocalPath; if (!Directory.Exists(fileShare)) { // download path does not exist, log and fall back var parenthPath = new Uri(buildArtifact.Resource.DownloadUrl).LocalPath; executionContext.Output(StringUtil.Loc("RMArtifactNameDirectoryNotFound", fileShare, parenthPath)); fileShare = parenthPath; } } if (!Directory.Exists(fileShare)) { // download path does not exist, raise exception throw new ArtifactDownloadException(StringUtil.Loc("RMArtifactDirectoryNotFoundError", fileShare)); } executionContext.Output(StringUtil.Loc("RMDownloadingArtifactFromFileShare", fileShare)); var fileShareArtifact = new FileShareArtifact(); await fileShareArtifact.DownloadArtifactAsync(executionContext, HostContext, artifactDefinition, fileShare, downloadFolderPath); } else if (buildArtifactDetails != null && string.Equals(buildArtifact.Resource.Type, WellKnownArtifactResourceTypes.Container, StringComparison.OrdinalIgnoreCase)) { executionContext.Output("Artifact Type: ServerDrop"); // Get containerId and rootLocation for the artifact #/922702/drop string[] parts = buildArtifact.Resource.Data.Split(new[] { '/' }, 3); if (parts.Length < 3) { throw new ArtifactDownloadException(StringUtil.Loc("RMArtifactContainerDetailsNotFoundError", buildArtifact.Name)); } int containerId; string rootLocation = parts[2]; if (!int.TryParse(parts[1], out containerId)) { throw new ArtifactDownloadException(StringUtil.Loc("RMArtifactContainerDetailsInvaidError", buildArtifact.Name)); } IContainerProvider containerProvider = new ContainerProviderFactory(buildArtifactDetails, rootLocation, containerId, executionContext).GetContainerProvider( WellKnownArtifactResourceTypes.Container); string rootDestinationDir = Path.Combine(localFolderPath, rootLocation); var containerFetchEngineOptions = new ContainerFetchEngineOptions { ParallelDownloadLimit = 4, CancellationToken = executionContext.CancellationToken }; using (var engine = new ContainerFetchEngine.ContainerFetchEngine(containerProvider, rootLocation, rootDestinationDir)) { engine.ContainerFetchEngineOptions = containerFetchEngineOptions; engine.ExecutionLogger = new ExecutionLogger(executionContext); await engine.FetchAsync(); } } else { executionContext.Warning(StringUtil.Loc("RMArtifactTypeNotSupported", buildArtifact.Resource.Type)); } }
private async Task DownloadArtifacts(IExecutionContext executionContext, IList <AgentArtifactDefinition> agentArtifactDefinitions, string artifactsWorkingFolder) { Trace.Entering(); CreateArtifactsFolder(executionContext, artifactsWorkingFolder); executionContext.Output(StringUtil.Loc("RMDownloadingArtifact")); foreach (AgentArtifactDefinition agentArtifactDefinition in agentArtifactDefinitions) { // We don't need to check if its old style artifact anymore. All the build data has been fixed and all the build artifact has Alias now. ArgUtil.NotNullOrEmpty(agentArtifactDefinition.Alias, nameof(agentArtifactDefinition.Alias)); var extensionManager = HostContext.GetService <IExtensionManager>(); IArtifactExtension extension = (extensionManager.GetExtensions <IArtifactExtension>()).FirstOrDefault(x => agentArtifactDefinition.ArtifactType == x.ArtifactType); if (extension == null) { throw new InvalidOperationException(StringUtil.Loc("RMArtifactTypeNotSupported", agentArtifactDefinition.ArtifactType)); } Trace.Info($"Found artifact extension of type {extension.ArtifactType}"); executionContext.Output(StringUtil.Loc("RMStartArtifactsDownload")); ArtifactDefinition artifactDefinition = ConvertToArtifactDefinition(agentArtifactDefinition, executionContext, extension); executionContext.Output(StringUtil.Loc("RMArtifactDownloadBegin", agentArtifactDefinition.Alias, agentArtifactDefinition.ArtifactType)); // Get the local path where this artifact should be downloaded. string downloadFolderPath = Path.GetFullPath(Path.Combine(artifactsWorkingFolder, agentArtifactDefinition.Alias ?? string.Empty)); // download the artifact to this path. RetryExecutor retryExecutor = new RetryExecutor(); retryExecutor.ShouldRetryAction = (ex) => { executionContext.Output(StringUtil.Loc("RMErrorDuringArtifactDownload", ex)); bool retry = true; if (ex is ArtifactDownloadException) { retry = false; } else { executionContext.Output(StringUtil.Loc("RMRetryingArtifactDownload")); Trace.Warning(ex.ToString()); } return(retry); }; await retryExecutor.ExecuteAsync( async() => { var releaseFileSystemManager = HostContext.GetService <IReleaseFileSystemManager>(); executionContext.Output(StringUtil.Loc("RMEnsureArtifactFolderExistsAndIsClean", downloadFolderPath)); releaseFileSystemManager.EnsureEmptyDirectory(downloadFolderPath, executionContext.CancellationToken); await extension.DownloadAsync(executionContext, artifactDefinition, downloadFolderPath); }); executionContext.Output(StringUtil.Loc("RMArtifactDownloadFinished", agentArtifactDefinition.Alias)); } executionContext.Output(StringUtil.Loc("RMArtifactsDownloadFinished")); }
public async Task DownloadCommitsAsync(IExecutionContext context, ArtifactDefinition artifactDefinition, string commitsWorkFolder) { ArgUtil.NotNull(context, nameof(context)); ArgUtil.NotNull(artifactDefinition, nameof(artifactDefinition)); Trace.Entering(); var jenkinsDetails = artifactDefinition.Details as JenkinsArtifactDetails; int startJobId = 0, endJobId = 0; if (!string.IsNullOrEmpty(jenkinsDetails.EndCommitArtifactVersion)) { if (int.TryParse(jenkinsDetails.EndCommitArtifactVersion, out endJobId)) { context.Output(StringUtil.Loc("RMDownloadingCommits")); if (int.TryParse(jenkinsDetails.StartCommitArtifactVersion, out startJobId)) { if (startJobId < endJobId) { context.Output(StringUtil.Loc("DownloadingJenkinsCommitsBetween", startJobId, endJobId)); } else if (startJobId > endJobId) { context.Output(StringUtil.Loc("JenkinsRollbackDeployment", startJobId, endJobId)); // swap the job IDs to fetch the roll back commits int swap = startJobId; startJobId = endJobId; endJobId = swap; } else if (startJobId == endJobId) { context.Output(StringUtil.Loc("JenkinsNoCommitsToFetch")); return; } } else { context.Debug(StringUtil.Loc("JenkinsDownloadingChangeFromCurrentBuild")); } try { IEnumerable <Change> changes = await DownloadCommits(context, jenkinsDetails, startJobId, endJobId); if (changes.Any()) { string commitsFileName = GetCommitsFileName(jenkinsDetails.Alias); string commitsFilePath = Path.Combine(commitsWorkFolder, commitsFileName); context.Debug($"Commits will be written to {commitsFilePath}"); WriteCommitsToFile(context, changes, commitsFilePath); context.Debug($"Commits written to {commitsFilePath}"); context.QueueAttachFile(CoreAttachmentType.FileAttachment, commitsFileName, commitsFilePath); } } catch (SocketException ex) { context.AddIssue(new Issue { Type = IssueType.Error, Message = $"SocketException occurred. { ex.Message }." + $"Verify whether you have (network) access to { jenkinsDetails.Url }. URLs the agent need communicate with - { BlobStoreWarningInfoProvider.GetAllowListLinkForCurrentPlatform() }." }); return; } catch (Exception ex) { context.AddIssue(new Issue { Type = IssueType.Warning, Message = StringUtil.Loc("DownloadingJenkinsCommitsFailedWithException", jenkinsDetails.Alias, ex.ToString()) }); return; } } else { context.AddIssue(new Issue { Type = IssueType.Warning, Message = StringUtil.Loc("JenkinsCommitsInvalidEndJobId", jenkinsDetails.EndCommitArtifactVersion, jenkinsDetails.Alias) }); return; } } else { context.Debug("No commit details found in the agent artifact. Not downloading the commits"); } }
private async Task DownloadArtifactAsync( IExecutionContext executionContext, ServerBuildArtifact buildArtifact, ArtifactDefinition artifactDefinition, string localFolderPath, BuildHttpClient buildClient, XamlBuildHttpClient xamlBuildClient, DefinitionType definitionType, int buildId) { var downloadFolderPath = Path.Combine(localFolderPath, buildArtifact.Name); var buildArtifactDetails = artifactDefinition.Details as BuildArtifactDetails; if ((buildArtifact.Resource.Type == null && buildArtifact.Id == 0) || // bug on build API Bug 378900 string.Equals(buildArtifact.Resource.Type, WellKnownArtifactResourceTypes.FilePath, StringComparison.OrdinalIgnoreCase)) { executionContext.Output("Artifact Type: FileShare"); string fileShare; if (buildArtifact.Id == 0) { fileShare = new Uri(buildArtifact.Resource.DownloadUrl).LocalPath; } else { fileShare = new Uri(Path.Combine(buildArtifact.Resource.DownloadUrl, buildArtifact.Name)).LocalPath; if (!Directory.Exists(fileShare)) { // download path does not exist, log and fall back executionContext.Output(StringUtil.Loc("RMArtifactNameDirectoryNotFound", fileShare, buildArtifact.Resource.DownloadUrl)); fileShare = new Uri(buildArtifact.Resource.DownloadUrl).LocalPath; } } if (!Directory.Exists(fileShare)) { // download path does not exist, raise exception throw new ArtifactDownloadException(StringUtil.Loc("RMArtifactDirectoryNotFoundError", fileShare)); } var fileShareArtifact = new FileShareArtifact(); await fileShareArtifact.DownloadArtifactAsync(executionContext, HostContext, artifactDefinition, fileShare, downloadFolderPath); } else if (string.Equals(buildArtifact.Resource.Type, WellKnownArtifactResourceTypes.Container, StringComparison.OrdinalIgnoreCase)) { executionContext.Output("Artifact Type: ServerDrop"); // TODO:Get VssBinFetchclient and get away from zipstream downloader Stream contentStream; if (definitionType == DefinitionType.Xaml) { contentStream = await xamlBuildClient.GetArtifactContentZipAsync(buildArtifactDetails.Project, buildId, buildArtifact.Name); } else { contentStream = await buildClient.GetArtifactContentZipAsync(buildArtifactDetails.Project, buildId, buildArtifact.Name); } var zipStreamDownloader = HostContext.GetService <IZipStreamDownloader>(); string artifactRootFolder = StringUtil.Format("/{0}", buildArtifact.Name); await zipStreamDownloader.DownloadFromStream(contentStream, artifactRootFolder, buildArtifactDetails.RelativePath, downloadFolderPath); } else { executionContext.Warning(StringUtil.Loc("RMArtifactTypeNotSupported", buildArtifact.Resource.Type)); } }
public async Task DownloadAsync( IExecutionContext executionContext, ArtifactDefinition artifactDefinition, string localFolderPath) { ArgUtil.NotNull(artifactDefinition, nameof(artifactDefinition)); ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNullOrEmpty(localFolderPath, nameof(localFolderPath)); var jenkinsDetails = artifactDefinition.Details as JenkinsArtifactDetails; executionContext.Output(StringUtil.Loc("RMGotJenkinsArtifactDetails")); executionContext.Output(StringUtil.Loc("RMJenkinsJobName", jenkinsDetails.JobName)); executionContext.Output(StringUtil.Loc("RMJenkinsBuildId", jenkinsDetails.BuildId)); IGenericHttpClient client = HostContext.GetService <IGenericHttpClient>(); if (!IsValidBuild(client, jenkinsDetails)) { throw new ArtifactDownloadException(StringUtil.Loc("RMJenkinsInvalidBuild", jenkinsDetails.BuildId)); } Stream downloadedStream = null; string downloadArtifactsUrl = string.Format( CultureInfo.InvariantCulture, "{0}/job/{1}/{2}/artifact/{3}/*zip*/", jenkinsDetails.Url, jenkinsDetails.JobName, jenkinsDetails.BuildId, jenkinsDetails.RelativePath); executionContext.Output(StringUtil.Loc("RMPrepareToGetFromJenkinsServer")); HttpResponseMessage response = client.GetAsync(downloadArtifactsUrl, jenkinsDetails.AccountName, jenkinsDetails.AccountPassword, jenkinsDetails.AcceptUntrustedCertificates).Result; if (response.IsSuccessStatusCode) { downloadedStream = response.Content.ReadAsStreamAsync().Result; } else if (response.StatusCode == HttpStatusCode.NotFound) { executionContext.Warning(StringUtil.Loc("RMJenkinsNoArtifactsFound", jenkinsDetails.BuildId)); return; } else { throw new ArtifactDownloadException(StringUtil.Loc("RMDownloadArtifactUnexpectedError")); } var parentFolder = GetParentFolderName(jenkinsDetails.RelativePath); Trace.Info($"Found parentFolder {parentFolder} for relative path {jenkinsDetails.RelativePath}"); executionContext.Output(StringUtil.Loc("RMDownloadingJenkinsArtifacts")); var zipStreamDownloader = HostContext.GetService <IZipStreamDownloader>(); await zipStreamDownloader.DownloadFromStream( executionContext, downloadedStream, string.IsNullOrEmpty(parentFolder)? "archive" : string.Empty, parentFolder, localFolderPath); }
public async Task DownloadAsync(IExecutionContext executionContext, ArtifactDefinition artifactDefinition, string localFolderPath) { ArgUtil.NotNull(artifactDefinition, nameof(artifactDefinition)); ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNullOrEmpty(localFolderPath, nameof(localFolderPath)); int buildId = Convert.ToInt32(artifactDefinition.Version, CultureInfo.InvariantCulture); if (buildId <= 0) { throw new ArgumentException("artifactDefinition.Version"); } var buildArtifactDetails = artifactDefinition.Details as BuildArtifactDetails; if (buildArtifactDetails == null) { throw new ArgumentException("artifactDefinition.Details"); } // Get the list of available artifacts from build. executionContext.Output(StringUtil.Loc("RMPreparingToGetBuildArtifactList")); var vssConnection = VssUtil.CreateConnection(buildArtifactDetails.TfsUrl, buildArtifactDetails.Credentials); var buildClient = vssConnection.GetClient <BuildHttpClient>(); var xamlBuildClient = vssConnection.GetClient <XamlBuildHttpClient>(); List <ServerBuildArtifact> buildArtifacts = null; EnsureVersionBelongsToLinkedDefinition(artifactDefinition, buildClient, xamlBuildClient); try { buildArtifacts = await buildClient.GetArtifactsAsync(buildArtifactDetails.Project, buildId); } catch (BuildNotFoundException) { buildArtifacts = await xamlBuildClient.GetArtifactsAsync(buildArtifactDetails.Project, buildId); } // No artifacts found in the build, add warning. if (buildArtifacts == null || !buildArtifacts.Any()) { executionContext.Warning(StringUtil.Loc("RMNoBuildArtifactsFound", buildId)); return; } // DownloadFromStream each of the artifact sequentially. // TODO: Should we download them parallely? foreach (ServerBuildArtifact buildArtifact in buildArtifacts) { if (Match(buildArtifact, artifactDefinition)) { executionContext.Output(StringUtil.Loc("RMPreparingToDownload", buildArtifact.Name)); await this.DownloadArtifactAsync(executionContext, buildArtifact, artifactDefinition, localFolderPath); } else { executionContext.Warning(StringUtil.Loc("RMArtifactMatchNotFound", buildArtifact.Name)); } } }
public async Task DownloadCommitsAsync(IExecutionContext context, ArtifactDefinition artifactDefinition, string commitsWorkFolder) { Trace.Entering(); var jenkinsDetails = artifactDefinition.Details as JenkinsArtifactDetails; int startJobId = 0, endJobId = 0; if (!string.IsNullOrEmpty(jenkinsDetails.EndCommitArtifactVersion)) { if (int.TryParse(jenkinsDetails.EndCommitArtifactVersion, out endJobId)) { context.Output(StringUtil.Loc("RMDownloadingCommits")); if (int.TryParse(jenkinsDetails.StartCommitArtifactVersion, out startJobId)) { if (startJobId < endJobId) { context.Output(StringUtil.Loc("DownloadingJenkinsCommitsBetween", startJobId, endJobId)); } else if (startJobId > endJobId) { context.Output(StringUtil.Loc("JenkinsRollbackDeployment", startJobId, endJobId)); // swap the job IDs to fetch the roll back commits int swap = startJobId; startJobId = endJobId; endJobId = swap; } else if (startJobId == endJobId) { context.Output(StringUtil.Loc("JenkinsNoCommitsToFetch")); return; } } else { context.Debug(StringUtil.Loc("JenkinsDownloadingChangeFromCurrentBuild")); } try { IEnumerable <Change> changes = await DownloadCommits(context, jenkinsDetails, startJobId, endJobId); if (changes.Any()) { string commitsFileName = GetCommitsFileName(jenkinsDetails.Alias); string commitsFilePath = Path.Combine(commitsWorkFolder, commitsFileName); context.Debug($"Commits will be written to {commitsFilePath}"); WriteCommitsToFile(context, changes, commitsFilePath); context.Debug($"Commits written to {commitsFilePath}"); context.QueueAttachFile(CoreAttachmentType.FileAttachment, commitsFileName, commitsFilePath); } } catch (Exception ex) { context.AddIssue(new Issue { Type = IssueType.Warning, Message = StringUtil.Loc("DownloadingJenkinsCommitsFailedWithException", jenkinsDetails.Alias, ex.ToString()) }); return; } } else { context.AddIssue(new Issue { Type = IssueType.Warning, Message = StringUtil.Loc("JenkinsCommitsInvalidEndJobId", jenkinsDetails.EndCommitArtifactVersion, jenkinsDetails.Alias) }); return; } } else { context.Debug("No commit details found in the agent artifact. Not downloading the commits"); } }
// This is only used by build artifact. This isn't a officially supported artifact type in RM public async Task DownloadArtifactAsync(IExecutionContext executionContext, IHostContext hostContext, ArtifactDefinition artifactDefinition, string dropLocation, string localFolderPath) { ArgUtil.NotNull(artifactDefinition, nameof(artifactDefinition)); ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNullOrEmpty(localFolderPath, nameof(localFolderPath)); ArgUtil.NotNullOrEmpty(dropLocation, nameof(dropLocation)); bool disableRobocopy = executionContext.Variables.GetBoolean(Constants.Variables.Release.DisableRobocopy) ?? false; if (disableRobocopy == false) { await DownloadArtifactUsingRobocopyAsync(executionContext, hostContext, artifactDefinition, dropLocation, localFolderPath); } else { await DownloadArtifactUsingFileSystemManagerAsync(executionContext, hostContext, artifactDefinition, dropLocation, localFolderPath); } }
// This is only used by build artifact. This isn't a officially supported artifact type in RM public async Task DownloadArtifactAsync(IExecutionContext executionContext, IHostContext hostContext, ArtifactDefinition artifactDefinition, string dropLocation, string localFolderPath) { ArgUtil.NotNull(artifactDefinition, nameof(artifactDefinition)); ArgUtil.NotNull(executionContext, nameof(executionContext)); ArgUtil.NotNullOrEmpty(localFolderPath, nameof(localFolderPath)); ArgUtil.NotNullOrEmpty(dropLocation, nameof(dropLocation)); var trimChars = new[] { '\\', '/' }; var relativePath = artifactDefinition.Details.RelativePath; // If user has specified a relative folder in the drop, change the drop location itself. dropLocation = Path.Combine(dropLocation.TrimEnd(trimChars), relativePath.Trim(trimChars)); var fileSystemManager = hostContext.CreateService <IReleaseFileSystemManager>(); List <string> filePaths = new DirectoryInfo(dropLocation).EnumerateFiles("*", SearchOption.AllDirectories) .Select(path => path.FullName) .ToList(); if (filePaths.Any()) { foreach (var filePath in filePaths) { string fullPath = Path.GetFullPath(filePath); if (File.Exists(fullPath)) { string filePathRelativeToDrop = filePath.Replace(dropLocation, string.Empty).Trim(trimChars); using (StreamReader fileReader = fileSystemManager.GetFileReader(filePath)) { await fileSystemManager.WriteStreamToFile( fileReader.BaseStream, Path.Combine(localFolderPath, filePathRelativeToDrop), executionContext.CancellationToken); } } else { executionContext.Warning(StringUtil.Loc("FileNotFound", fullPath)); } } } else { executionContext.Warning(StringUtil.Loc("RMArtifactEmpty")); } }
private async Task DownloadArtifactUsingRobocopyAsync(IExecutionContext executionContext, IHostContext hostContext, ArtifactDefinition artifactDefinition, string dropLocation, string downloadFolderPath) { int? robocopyMT = executionContext.Variables.GetInt(Constants.Variables.Release.RobocopyMT); bool verbose = executionContext.Variables.GetBoolean(Constants.Variables.System.Debug) ?? false; if (robocopyMT != null) { if (robocopyMT < 1) { robocopyMT = 1; } else if (robocopyMT > 128) { robocopyMT = 128; } } executionContext.Output(StringUtil.Loc("RMDownloadingArtifactUsingRobocopy")); using (var processInvoker = hostContext.CreateService <IProcessInvoker>()) { // Save STDOUT from worker, worker will use STDOUT report unhandle exception. processInvoker.OutputDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stdout) { if (!string.IsNullOrEmpty(stdout.Data)) { executionContext.Output(stdout.Data); } }; // Save STDERR from worker, worker will use STDERR on crash. processInvoker.ErrorDataReceived += delegate(object sender, ProcessDataReceivedEventArgs stderr) { if (!string.IsNullOrEmpty(stderr.Data)) { executionContext.Error(stderr.Data); } }; var trimChars = new[] { '\\', '/' }; var relativePath = artifactDefinition.Details.RelativePath; dropLocation = Path.Combine(dropLocation.TrimEnd(trimChars), relativePath.Trim(trimChars)); downloadFolderPath = downloadFolderPath.TrimEnd(trimChars); string robocopyArguments = "\"" + dropLocation + "\" \"" + downloadFolderPath + "\" /E /Z /NP /R:3"; if (verbose != true) { robocopyArguments = robocopyArguments + " /NDL /NFL"; } if (robocopyMT != null) { robocopyArguments = robocopyArguments + " /MT:" + robocopyMT; } int exitCode = await processInvoker.ExecuteAsync( workingDirectory : "", fileName : "robocopy", arguments : robocopyArguments, environment : null, requireExitCodeZero : false, outputEncoding : null, killProcessOnCancel : true, cancellationToken : executionContext.CancellationToken); executionContext.Output(StringUtil.Loc("RMRobocopyBasedArtifactDownloadExitCode", exitCode)); if (exitCode >= 8) { throw new ArtifactDownloadException(StringUtil.Loc("RMRobocopyBasedArtifactDownloadFailed", exitCode)); } } }