public async Task DownloadMultipleArtifactsAsync(PipelineArtifactDownloadParameters downloadParameters, IEnumerable <BuildArtifact> buildArtifacts, CancellationToken cancellationToken) { DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient( this.context, this.connection, cancellationToken, out BlobStoreClientTelemetry clientTelemetry); using (clientTelemetry) { var artifactNameAndManifestIds = buildArtifacts.ToDictionary( keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data)); // 2) download to the target path var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds( artifactNameAndManifestIds, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters, minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName); PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) => new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await dedupManifestClient.DownloadAsync(options, cancellationToken); }); // Send results to CustomerIntelligence this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord); } }
public async Task DownloadSingleArtifactAsync(PipelineArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken) { DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient( this.context, this.connection, cancellationToken, out BlobStoreClientTelemetry clientTelemetry); using (clientTelemetry) { var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data); var options = DownloadDedupManifestArtifactOptions.CreateWithManifestId( manifestId, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters); PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) => new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await dedupManifestClient.DownloadAsync(options, cancellationToken); }); // Send results to CustomerIntelligence this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord); } }
private async Task DownloadPipelineCacheAsync( AgentTaskPluginExecutionContext context, DedupManifestArtifactClient dedupManifestClient, DedupIdentifier manifestId, string targetDirectory, string contentFormat, CancellationToken cancellationToken) { if (contentFormat == ContentFormatConstants.SingleTar) { string manifestPath = Path.Combine(Path.GetTempPath(), $"{nameof(DedupManifestArtifactClient)}.{Path.GetRandomFileName()}.manifest"); await dedupManifestClient.DownloadFileToPathAsync(manifestId, manifestPath, proxyUri : null, cancellationToken : cancellationToken); Manifest manifest = JsonSerializer.Deserialize <Manifest>(File.ReadAllText(manifestPath)); await TarUtils.DownloadAndExtractTarAsync(context, manifest, dedupManifestClient, targetDirectory, cancellationToken); try { if (File.Exists(manifestPath)) { File.Delete(manifestPath); } } catch {} } else { DownloadDedupManifestArtifactOptions options = DownloadDedupManifestArtifactOptions.CreateWithManifestId( manifestId, targetDirectory, proxyUri: null, minimatchPatterns: null); await dedupManifestClient.DownloadAsync(options, cancellationToken); } }
private async Task DownloadPipelineCacheAsync( AgentTaskPluginExecutionContext context, DedupManifestArtifactClient dedupManifestClient, DedupIdentifier manifestId, string[] pathSegments, string workspaceRoot, ContentFormat contentFormat, CancellationToken cancellationToken) { if (contentFormat == ContentFormat.SingleTar) { string manifestPath = Path.Combine(Path.GetTempPath(), $"{nameof(DedupManifestArtifactClient)}.{Path.GetRandomFileName()}.manifest"); await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { await dedupManifestClient.DownloadFileToPathAsync(manifestId, manifestPath, proxyUri: null, cancellationToken: cancellationToken); }, maxRetries : 3, tracer : tracer, canRetryDelegate : e => true, context : nameof(DownloadPipelineCacheAsync), cancellationToken : cancellationToken, continueOnCapturedContext : false); Manifest manifest = JsonSerializer.Deserialize <Manifest>(File.ReadAllText(manifestPath)); var(tarWorkingDirectory, _) = GetTarWorkingDirectory(pathSegments, workspaceRoot); await TarUtils.DownloadAndExtractTarAsync(context, manifest, dedupManifestClient, tarWorkingDirectory, cancellationToken); try { if (File.Exists(manifestPath)) { File.Delete(manifestPath); } } catch { } } else { DownloadDedupManifestArtifactOptions options = DownloadDedupManifestArtifactOptions.CreateWithManifestId( manifestId, pathSegments[0], proxyUri: null, minimatchPatterns: null); await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { await dedupManifestClient.DownloadAsync(options, cancellationToken); }, maxRetries : 3, tracer : tracer, canRetryDelegate : e => true, context : nameof(DownloadPipelineCacheAsync), cancellationToken : cancellationToken, continueOnCapturedContext : false); } }
public async Task DownloadSingleArtifactAsync(PipelineArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken) { var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data); var options = DownloadDedupManifestArtifactOptions.CreateWithManifestId( manifestId, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters); await dedupManifestArtifactClient.DownloadAsync(options, cancellationToken); }
public async Task DownloadMultipleArtifactsAsync(PipelineArtifactDownloadParameters downloadParameters, IEnumerable <BuildArtifact> buildArtifacts, CancellationToken cancellationToken) { var artifactNameAndManifestIds = buildArtifacts.ToDictionary( keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data)); // 2) download to the target path var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds( artifactNameAndManifestIds, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters, minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName); await dedupManifestArtifactClient.DownloadAsync(options, cancellationToken); }
private Task DownloadPipelineCacheAsync( DedupManifestArtifactClient dedupManifestClient, DedupIdentifier manifestId, string targetDirectory, CancellationToken cancellationToken) { DownloadDedupManifestArtifactOptions options = DownloadDedupManifestArtifactOptions.CreateWithManifestId( manifestId, targetDirectory, proxyUri: null, minimatchPatterns: null); return(dedupManifestClient.DownloadAsync(options, cancellationToken)); }
public async Task DownloadMultipleArtifactsAsync(ArtifactDownloadParameters downloadParameters, IEnumerable <BuildArtifact> buildArtifacts, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context) { var(dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClientAsync( this.context.IsSystemDebugTrue(), (str) => this.context.Output(str), this.connection, cancellationToken); using (clientTelemetry) { var artifactNameAndManifestIds = buildArtifacts.ToDictionary( keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data)); // 2) download to the target path var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds( artifactNameAndManifestIds, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters, minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName); PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) => new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { await dedupManifestClient.DownloadAsync(options, cancellationToken); }, maxRetries: 3, tracer: tracer, canRetryDelegate: e => true, context: nameof(DownloadMultipleArtifactsAsync), cancellationToken: cancellationToken, continueOnCapturedContext: false); }); // Send results to CustomerIntelligence this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord); } }
public async Task DownloadSingleArtifactAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context) { var(dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClientAsync( this.context.IsSystemDebugTrue(), (str) => this.context.Output(str), this.connection, cancellationToken); using (clientTelemetry) { var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data); var options = DownloadDedupManifestArtifactOptions.CreateWithManifestId( manifestId, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters); PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) => new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { await dedupManifestClient.DownloadAsync(options, cancellationToken); }, maxRetries: 3, tracer: tracer, canRetryDelegate: e => true, context: nameof(DownloadSingleArtifactAsync), cancellationToken: cancellationToken, continueOnCapturedContext: false); }); // Send results to CustomerIntelligence this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord); } }
// Download with minimatch patterns, V1. internal async Task DownloadAsync( AgentTaskPluginExecutionContext context, PipelineArtifactDownloadParameters downloadParameters, DownloadOptions downloadOptions, CancellationToken cancellationToken) { VssConnection connection = context.VssConnection; BlobStoreClientTelemetry clientTelemetry; DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry); BuildServer buildHelper = new BuildServer(connection); using (clientTelemetry) { // download all pipeline artifacts if artifact name is missing if (downloadOptions == DownloadOptions.MultiDownload) { List <BuildArtifact> artifacts; if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId) { artifacts = await buildHelper.GetArtifactsAsync(downloadParameters.ProjectId, downloadParameters.PipelineId, cancellationToken); } else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName) { if (string.IsNullOrEmpty(downloadParameters.ProjectName)) { throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!"); } else { artifacts = await buildHelper.GetArtifactsWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, cancellationToken); } } else { throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!"); } IEnumerable <BuildArtifact> pipelineArtifacts = artifacts.Where(a => string.Equals(a.Resource.Type, PipelineArtifactConstants.PipelineArtifact, StringComparison.OrdinalIgnoreCase)); if (pipelineArtifacts.Count() == 0) { throw new ArgumentException("Could not find any pipeline artifacts in the build."); } else { context.Output(StringUtil.Loc("DownloadingMultiplePipelineArtifacts", pipelineArtifacts.Count())); var artifactNameAndManifestIds = pipelineArtifacts.ToDictionary( keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data)); // 2) download to the target path var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds( artifactNameAndManifestIds, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters, minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName); PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) => new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await dedupManifestClient.DownloadAsync(options, cancellationToken); }); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord); } } else if (downloadOptions == DownloadOptions.SingleDownload) { // 1) get manifest id from artifact data BuildArtifact buildArtifact; if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId) { buildArtifact = await buildHelper.GetArtifact(downloadParameters.ProjectId, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken); } else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName) { if (string.IsNullOrEmpty(downloadParameters.ProjectName)) { throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!"); } else { buildArtifact = await buildHelper.GetArtifactWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken); } } else { throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!"); } var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data); var options = DownloadDedupManifestArtifactOptions.CreateWithManifestId( manifestId, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters); PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) => new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await dedupManifestClient.DownloadAsync(options, cancellationToken); }); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord); } else { throw new InvalidOperationException($"Invalid {nameof(downloadOptions)}!"); } } }