public async Task DownloadMultipleArtifactsAsync(PipelineArtifactDownloadParameters downloadParameters, IEnumerable <BuildArtifact> buildArtifacts, CancellationToken cancellationToken) { DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient( this.context, this.connection, cancellationToken, out BlobStoreClientTelemetry clientTelemetry); using (clientTelemetry) { var artifactNameAndManifestIds = buildArtifacts.ToDictionary( keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data)); // 2) download to the target path var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds( artifactNameAndManifestIds, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters, minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName); PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) => new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await dedupManifestClient.DownloadAsync(options, cancellationToken); }); // Send results to CustomerIntelligence this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord); } }
public async Task DownloadMultipleArtifactsAsync(PipelineArtifactDownloadParameters downloadParameters, IEnumerable <BuildArtifact> buildArtifacts, CancellationToken cancellationToken) { var artifactNameAndManifestIds = buildArtifacts.ToDictionary( keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data)); // 2) download to the target path var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds( artifactNameAndManifestIds, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters, minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName); await dedupManifestArtifactClient.DownloadAsync(options, cancellationToken); }
public async Task DownloadMultipleArtifactsAsync(ArtifactDownloadParameters downloadParameters, IEnumerable <BuildArtifact> buildArtifacts, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context) { var(dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClientAsync( this.context.IsSystemDebugTrue(), (str) => this.context.Output(str), this.connection, cancellationToken); using (clientTelemetry) { var artifactNameAndManifestIds = buildArtifacts.ToDictionary( keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data)); // 2) download to the target path var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds( artifactNameAndManifestIds, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters, minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName); PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) => new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { await dedupManifestClient.DownloadAsync(options, cancellationToken); }, maxRetries: 3, tracer: tracer, canRetryDelegate: e => true, context: nameof(DownloadMultipleArtifactsAsync), cancellationToken: cancellationToken, continueOnCapturedContext: false); }); // Send results to CustomerIntelligence this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord); } }
// Download with minimatch patterns, V1. internal async Task DownloadAsync( AgentTaskPluginExecutionContext context, PipelineArtifactDownloadParameters downloadParameters, DownloadOptions downloadOptions, CancellationToken cancellationToken) { VssConnection connection = context.VssConnection; BlobStoreClientTelemetry clientTelemetry; DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry); BuildServer buildHelper = new BuildServer(connection); using (clientTelemetry) { // download all pipeline artifacts if artifact name is missing if (downloadOptions == DownloadOptions.MultiDownload) { List <BuildArtifact> artifacts; if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId) { artifacts = await buildHelper.GetArtifactsAsync(downloadParameters.ProjectId, downloadParameters.PipelineId, cancellationToken); } else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName) { if (string.IsNullOrEmpty(downloadParameters.ProjectName)) { throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!"); } else { artifacts = await buildHelper.GetArtifactsWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, cancellationToken); } } else { throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!"); } IEnumerable <BuildArtifact> pipelineArtifacts = artifacts.Where(a => string.Equals(a.Resource.Type, PipelineArtifactConstants.PipelineArtifact, StringComparison.OrdinalIgnoreCase)); if (pipelineArtifacts.Count() == 0) { throw new ArgumentException("Could not find any pipeline artifacts in the build."); } else { context.Output(StringUtil.Loc("DownloadingMultiplePipelineArtifacts", pipelineArtifacts.Count())); var artifactNameAndManifestIds = pipelineArtifacts.ToDictionary( keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data)); // 2) download to the target path var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds( artifactNameAndManifestIds, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters, minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName); PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) => new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await dedupManifestClient.DownloadAsync(options, cancellationToken); }); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord); } } else if (downloadOptions == DownloadOptions.SingleDownload) { // 1) get manifest id from artifact data BuildArtifact buildArtifact; if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId) { buildArtifact = await buildHelper.GetArtifact(downloadParameters.ProjectId, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken); } else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName) { if (string.IsNullOrEmpty(downloadParameters.ProjectName)) { throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!"); } else { buildArtifact = await buildHelper.GetArtifactWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken); } } else { throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!"); } var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data); var options = DownloadDedupManifestArtifactOptions.CreateWithManifestId( manifestId, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters); PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) => new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await dedupManifestClient.DownloadAsync(options, cancellationToken); }); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord); } else { throw new InvalidOperationException($"Invalid {nameof(downloadOptions)}!"); } } }