// Download pipeline artifact from VSTS BlobStore service through BuildDropManager to a target path internal async Task DownloadAsync( AgentTaskPluginExecutionContext context, Guid projectId, int buildId, string artifactName, string targetDir, CancellationToken cancellationToken) { VssConnection connection = context.VssConnection; // 1) get manifest id from artifact data BuildServer buildHelper = new BuildServer(connection); BuildArtifact art = await buildHelper.GetArtifact(projectId, buildId, artifactName, cancellationToken); var manifestId = DedupIdentifier.Create(art.Resource.Data); // 2) download to the target path var httpclient = connection.GetClient <DedupStoreHttpClient>(); var tracer = new CallbackAppTraceSource(str => context.Output(str), System.Diagnostics.SourceLevels.Information); httpclient.SetTracer(tracer); var client = new DedupStoreClientWithDataport(httpclient, maxParallelism: 16 * Environment.ProcessorCount); var BuildDropManager = new BuildDropManager(client, tracer); await BuildDropManager.DownloadAsync(manifestId, targetDir, cancellationToken); }
// Download with minimatch patterns. internal async Task DownloadAsyncMinimatch( AgentTaskPluginExecutionContext context, Guid projectId, int buildId, string artifactName, string targetDir, string[] minimatchFilters, CancellationToken cancellationToken) { VssConnection connection = context.VssConnection; // 1) get manifest id from artifact data BuildServer buildHelper = new BuildServer(connection); BuildArtifact art = await buildHelper.GetArtifact(projectId, buildId, artifactName, cancellationToken); if (art.Resource.Type != "PipelineArtifact") { throw new ArgumentException($"The artifact is not of the type Pipeline Artifact. Unrecognized type: {art.Resource.Type}."); } var manifestId = DedupIdentifier.Create(art.Resource.Data); // 2) download to the target path var buildDropManager = this.GetBDM(context, connection); DownloadPipelineArtifactOptions options = DownloadPipelineArtifactOptions.CreateWithManifestId(manifestId, targetDir, proxyUri: null, minimatchPatterns: minimatchFilters); await buildDropManager.DownloadAsync(options, cancellationToken); }
// Download pipeline artifact from VSTS BlobStore service through BuildDropManager to a target path internal async Task DownloadAsync( AgentTaskPluginExecutionContext context, Guid projectId, int buildId, string artifactName, string targetDir, CancellationToken cancellationToken) { VssConnection connection = context.VssConnection; // 1) get manifest id from artifact data BuildServer buildHelper = new BuildServer(connection); BuildArtifact art = await buildHelper.GetArtifact(projectId, buildId, artifactName, cancellationToken); if (art.Resource.Type != "PipelineArtifact") { throw new ArgumentException("The artifact is not of the type Pipeline Artifact\n"); } var manifestId = DedupIdentifier.Create(art.Resource.Data); // 2) download to the target path var buildDropManager = this.GetBDM(context, connection); await buildDropManager.DownloadAsync(manifestId, targetDir, cancellationToken); }
// Download with minimatch patterns, V1. internal async Task DownloadAsync( AgentTaskPluginExecutionContext context, PipelineArtifactDownloadParameters downloadParameters, DownloadOptions downloadOptions, CancellationToken cancellationToken) { VssConnection connection = context.VssConnection; BlobStoreClientTelemetry clientTelemetry; DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, out clientTelemetry); BuildServer buildHelper = new BuildServer(connection); using (clientTelemetry) { // download all pipeline artifacts if artifact name is missing if (downloadOptions == DownloadOptions.MultiDownload) { List <BuildArtifact> artifacts; if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId) { artifacts = await buildHelper.GetArtifactsAsync(downloadParameters.ProjectId, downloadParameters.PipelineId, cancellationToken); } else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName) { if (string.IsNullOrEmpty(downloadParameters.ProjectName)) { throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!"); } else { artifacts = await buildHelper.GetArtifactsWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, cancellationToken); } } else { throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!"); } IEnumerable <BuildArtifact> pipelineArtifacts = artifacts.Where(a => a.Resource.Type == PipelineArtifactConstants.PipelineArtifact); if (pipelineArtifacts.Count() == 0) { throw new ArgumentException("Could not find any pipeline artifacts in the build."); } else { context.Output(StringUtil.Loc("DownloadingMultiplePipelineArtifacts", pipelineArtifacts.Count())); var artifactNameAndManifestIds = pipelineArtifacts.ToDictionary( keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data)); // 2) download to the target path var options = DownloadPipelineArtifactOptions.CreateWithMultiManifestIds( artifactNameAndManifestIds, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters); PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) => new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await dedupManifestClient.DownloadAsync(options, cancellationToken); }); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord); } } else if (downloadOptions == DownloadOptions.SingleDownload) { // 1) get manifest id from artifact data BuildArtifact buildArtifact; if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId) { buildArtifact = await buildHelper.GetArtifact(downloadParameters.ProjectId, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken); } else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName) { if (string.IsNullOrEmpty(downloadParameters.ProjectName)) { throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!"); } else { buildArtifact = await buildHelper.GetArtifactWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken); } } else { throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!"); } var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data); var options = DownloadPipelineArtifactOptions.CreateWithManifestId( manifestId, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters); PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) => new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await dedupManifestClient.DownloadAsync(options, cancellationToken); }); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord); } else { throw new InvalidOperationException($"Invalid {nameof(downloadOptions)}!"); } } }
// Download for version 2. This decision was made because version 1 is sealed and we didn't want to break any existing customers. internal async Task DownloadAsyncV2( AgentTaskPluginExecutionContext context, PipelineArtifactDownloadParameters downloadParameters, DownloadOptions downloadOptions, CancellationToken cancellationToken) { VssConnection connection = context.VssConnection; BuildServer buildHelper = new BuildServer(connection); // download all pipeline artifacts if artifact name is missing if (downloadOptions == DownloadOptions.MultiDownload) { List <BuildArtifact> artifacts; if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId) { artifacts = await buildHelper.GetArtifactsAsync(downloadParameters.ProjectId, downloadParameters.PipelineId, cancellationToken); } else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName) { if (string.IsNullOrEmpty(downloadParameters.ProjectName)) { throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!"); } else { artifacts = await buildHelper.GetArtifactsWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, cancellationToken); } } else { throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!"); } IEnumerable <BuildArtifact> buildArtifacts = artifacts.Where(a => a.Resource.Type == PipelineArtifactConstants.Container); IEnumerable <BuildArtifact> pipelineArtifacts = artifacts.Where(a => a.Resource.Type == PipelineArtifactConstants.PipelineArtifact); if (buildArtifacts.Any()) { FileContainerProvider provider = new FileContainerProvider(connection, this.CreateTracer(context)); await provider.DownloadMultipleArtifactsAsync(downloadParameters, buildArtifacts, cancellationToken); } if (pipelineArtifacts.Any()) { PipelineArtifactProvider provider = new PipelineArtifactProvider(context, connection, this.CreateTracer(context)); await provider.DownloadMultipleArtifactsAsync(downloadParameters, pipelineArtifacts, cancellationToken); } } else if (downloadOptions == DownloadOptions.SingleDownload) { // 1) get manifest id from artifact data BuildArtifact buildArtifact; if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId) { buildArtifact = await buildHelper.GetArtifact(downloadParameters.ProjectId, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken); } else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName) { if (string.IsNullOrEmpty(downloadParameters.ProjectName)) { throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!"); } else { buildArtifact = await buildHelper.GetArtifactWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken); } } else { throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!"); } ArtifactProviderFactory factory = new ArtifactProviderFactory(context, connection, this.CreateTracer(context)); IArtifactProvider provider = factory.GetProvider(buildArtifact); await provider.DownloadSingleArtifactAsync(downloadParameters, buildArtifact, cancellationToken); } else { throw new InvalidOperationException($"Invalid {nameof(downloadOptions)}!"); } }
// Download with minimatch patterns. internal async Task DownloadAsync( AgentTaskPluginExecutionContext context, PipelineArtifactDownloadParameters downloadParameters, CancellationToken cancellationToken) { VssConnection connection = context.VssConnection; var buildDropManager = this.CreateBulidDropManager(context, connection); BuildServer buildHelper = new BuildServer(connection); // download all pipeline artifacts if artifact name is missing if (string.IsNullOrEmpty(downloadParameters.ArtifactName)) { List <BuildArtifact> artifacts = await buildHelper.GetArtifactsAsync(downloadParameters.ProjectId, downloadParameters.BuildId, cancellationToken); IEnumerable <BuildArtifact> pipelineArtifacts = artifacts.Where(a => a.Resource.Type == PipelineArtifactTypeName); if (pipelineArtifacts.Count() == 0) { throw new ArgumentException("Could not find any pipeline artifacts in the build."); } else { context.Output(StringUtil.Loc("DownloadingMultiplePipelineArtifacts", pipelineArtifacts.Count())); foreach (BuildArtifact pipelineArtifact in pipelineArtifacts) { // each pipeline artifact will have its own subroot to avoid file collisions string pipelineArtifactRootPath = Path.Combine(downloadParameters.TargetDirectory, pipelineArtifact.Name); await DownloadPipelineArtifact( buildDropManager, pipelineArtifact, pipelineArtifactRootPath, downloadParameters.MinimatchFilters, cancellationToken); } } } else { // 1) get manifest id from artifact data BuildArtifact buildArtifact; if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId) { buildArtifact = await buildHelper.GetArtifact(downloadParameters.ProjectId, downloadParameters.BuildId, downloadParameters.ArtifactName, cancellationToken); } else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName) { if (string.IsNullOrEmpty(downloadParameters.ProjectName)) { throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!"); } else { buildArtifact = await buildHelper.GetArtifactWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.BuildId, downloadParameters.ArtifactName, cancellationToken); } } else { throw new InvalidOperationException("Unreachable code!"); } await DownloadPipelineArtifact( buildDropManager, buildArtifact, downloadParameters.TargetDirectory, downloadParameters.MinimatchFilters, cancellationToken); } }
// Download with minimatch patterns. internal async Task DownloadAsync( AgentTaskPluginExecutionContext context, PipelineArtifactDownloadParameters downloadParameters, DownloadOptions downloadOptions, CancellationToken cancellationToken) { VssConnection connection = context.VssConnection; var buildDropManager = this.CreateBulidDropManager(context, connection); BuildServer buildHelper = new BuildServer(connection); // download all pipeline artifacts if artifact name is missing if (downloadOptions == DownloadOptions.MultiDownload) { List <BuildArtifact> artifacts; if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId) { artifacts = await buildHelper.GetArtifactsAsync(downloadParameters.ProjectId, downloadParameters.PipelineId, cancellationToken); } else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName) { if (string.IsNullOrEmpty(downloadParameters.ProjectName)) { throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!"); } else { artifacts = await buildHelper.GetArtifactsWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, cancellationToken); } } else { throw new InvalidOperationException("Unreachable code!"); } IEnumerable <BuildArtifact> pipelineArtifacts = artifacts.Where(a => a.Resource.Type == PipelineArtifactTypeName); if (pipelineArtifacts.Count() == 0) { throw new ArgumentException("Could not find any pipeline artifacts in the build."); } else { context.Output(StringUtil.Loc("DownloadingMultiplePipelineArtifacts", pipelineArtifacts.Count())); var artifactNameAndManifestIds = pipelineArtifacts.ToDictionary( keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data)); // 2) download to the target path var options = DownloadPipelineArtifactOptions.CreateWithMultiManifestIds( artifactNameAndManifestIds, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters); await buildDropManager.DownloadAsync(options, cancellationToken); } } else if (downloadOptions == DownloadOptions.SingleDownload) { // 1) get manifest id from artifact data BuildArtifact buildArtifact; if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId) { buildArtifact = await buildHelper.GetArtifact(downloadParameters.ProjectId, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken); } else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName) { if (string.IsNullOrEmpty(downloadParameters.ProjectName)) { throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!"); } else { buildArtifact = await buildHelper.GetArtifactWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken); } } else { throw new InvalidOperationException("Unreachable code!"); } var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data); var options = DownloadPipelineArtifactOptions.CreateWithManifestId( manifestId, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters); await buildDropManager.DownloadAsync(options, cancellationToken); } else { throw new InvalidOperationException("Unreachable code!"); } }