private async Task DownloadPipelineCacheAsync( AgentTaskPluginExecutionContext context, DedupManifestArtifactClient dedupManifestClient, DedupIdentifier manifestId, string[] pathSegments, string workspaceRoot, ContentFormat contentFormat, CancellationToken cancellationToken) { if (contentFormat == ContentFormat.SingleTar) { string manifestPath = Path.Combine(Path.GetTempPath(), $"{nameof(DedupManifestArtifactClient)}.{Path.GetRandomFileName()}.manifest"); await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { await dedupManifestClient.DownloadFileToPathAsync(manifestId, manifestPath, proxyUri: null, cancellationToken: cancellationToken); }, maxRetries : 3, tracer : tracer, canRetryDelegate : e => true, context : nameof(DownloadPipelineCacheAsync), cancellationToken : cancellationToken, continueOnCapturedContext : false); Manifest manifest = JsonSerializer.Deserialize <Manifest>(File.ReadAllText(manifestPath)); var(tarWorkingDirectory, _) = GetTarWorkingDirectory(pathSegments, workspaceRoot); await TarUtils.DownloadAndExtractTarAsync(context, manifest, dedupManifestClient, tarWorkingDirectory, cancellationToken); try { if (File.Exists(manifestPath)) { File.Delete(manifestPath); } } catch { } } else { DownloadDedupManifestArtifactOptions options = DownloadDedupManifestArtifactOptions.CreateWithManifestId( manifestId, pathSegments[0], proxyUri: null, minimatchPatterns: null); await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { await dedupManifestClient.DownloadAsync(options, cancellationToken); }, maxRetries : 3, tracer : tracer, canRetryDelegate : e => true, context : nameof(DownloadPipelineCacheAsync), cancellationToken : cancellationToken, continueOnCapturedContext : false); } }
public async Task DownloadMultipleArtifactsAsync(PipelineArtifactDownloadParameters downloadParameters, IEnumerable <BuildArtifact> buildArtifacts, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context) { DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClient( this.context, this.connection, cancellationToken, out BlobStoreClientTelemetry clientTelemetry); using (clientTelemetry) { var artifactNameAndManifestIds = buildArtifacts.ToDictionary( keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data)); // 2) download to the target path var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds( artifactNameAndManifestIds, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters, minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName); PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) => new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { await dedupManifestClient.DownloadAsync(options, cancellationToken); }, maxRetries: 3, tracer: tracer, canRetryDelegate: e => true, context: nameof(DownloadMultipleArtifactsAsync), cancellationToken: cancellationToken, continueOnCapturedContext: false); }); // Send results to CustomerIntelligence this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord); } }
public async Task DownloadSingleArtifactAsync(PipelineArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context) { DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClient( this.context, this.connection, cancellationToken, out BlobStoreClientTelemetry clientTelemetry); using (clientTelemetry) { var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data); var options = DownloadDedupManifestArtifactOptions.CreateWithManifestId( manifestId, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters); PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) => new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { await dedupManifestClient.DownloadAsync(options, cancellationToken); }, maxRetries: 3, tracer: tracer, canRetryDelegate: e => true, context: nameof(DownloadSingleArtifactAsync), cancellationToken: cancellationToken, continueOnCapturedContext: false); }); // Send results to CustomerIntelligence this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord); } }
private async Task DownloadFileContainerAsync(Guid projectId, BuildArtifact artifact, string rootPath, IEnumerable <string> minimatchPatterns, CancellationToken cancellationToken, bool isSingleArtifactDownload = true) { var containerIdAndRoot = ParseContainerId(artifact.Resource.Data); var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, containerIdAndRoot.Item2); tracer.Info($"Start downloading FCS artifact- {artifact.Name}"); IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer); if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0) { items = this.GetFilteredItems(items, minimatcherFuncs); } if (!isSingleArtifactDownload && items.Any()) { Directory.CreateDirectory(rootPath); } var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder); Parallel.ForEach(folderItems, (folder) => { var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2); Directory.CreateDirectory(targetPath); }); var fileItems = items.Where(i => i.ItemType == ContainerItemType.File); var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>( async item => { var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2); var directory = Path.GetDirectoryName(targetPath); Directory.CreateDirectory(directory); await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken)) { tracer.Info($"Downloading: {targetPath}"); using (var targetStream = new FileStream(targetPath, FileMode.Create)) { await sourceStream.CopyToAsync(targetStream); } } }, maxRetries: 5, cancellationToken: cancellationToken, tracer: tracer, continueOnCapturedContext: false, canRetryDelegate: exception => exception is IOException, context: null ); }, new ExecutionDataflowBlockOptions() { BoundedCapacity = 5000, MaxDegreeOfParallelism = 8, CancellationToken = cancellationToken, }); await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken); }
private async Task DownloadFileContainerAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true) { var containerIdAndRoot = ParseContainerId(artifact.Resource.Data); var projectId = downloadParameters.ProjectId; var minimatchPatterns = downloadParameters.MinimatchFilters; var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, isShallow : false, includeBlobMetadata : true, containerIdAndRoot.Item2); tracer.Info($"Start downloading FCS artifact- {artifact.Name}"); IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer); if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0) { items = this.GetFilteredItems(items, minimatcherFuncs); } if (!isSingleArtifactDownload && items.Any()) { Directory.CreateDirectory(rootPath); } var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder); Parallel.ForEach(folderItems, (folder) => { var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); Directory.CreateDirectory(targetPath); }); var fileItems = items.Where(i => i.ItemType == ContainerItemType.File); var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>( async item => { var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); var directory = Path.GetDirectoryName(targetPath); Directory.CreateDirectory(directory); await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { tracer.Info($"Downloading: {targetPath}"); if (item.BlobMetadata != null) { await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, cancellationToken); } else { using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken)) using (var targetStream = new FileStream(targetPath, FileMode.Create)) { await sourceStream.CopyToAsync(targetStream); } } }, maxRetries: downloadParameters.RetryDownloadCount, cancellationToken: cancellationToken, tracer: tracer, continueOnCapturedContext: false, canRetryDelegate: exception => exception is IOException, context: null ); }, new ExecutionDataflowBlockOptions() { BoundedCapacity = 5000, MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit, CancellationToken = cancellationToken, }); await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken); // check files (will throw an exception if a file is corrupt) if (downloadParameters.CheckDownloadedFiles) { CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); } }
// Download with minimatch patterns, V1. internal async Task DownloadAsync( AgentTaskPluginExecutionContext context, PipelineArtifactDownloadParameters downloadParameters, DownloadOptions downloadOptions, CancellationToken cancellationToken) { VssConnection connection = context.VssConnection; BlobStoreClientTelemetry clientTelemetry; DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry); BuildServer buildServer = new BuildServer(connection); using (clientTelemetry) { // download all pipeline artifacts if artifact name is missing if (downloadOptions == DownloadOptions.MultiDownload) { List <BuildArtifact> artifacts; if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId) { artifacts = await buildServer.GetArtifactsAsync(downloadParameters.ProjectId, downloadParameters.PipelineId, cancellationToken); } else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName) { if (string.IsNullOrEmpty(downloadParameters.ProjectName)) { throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!"); } else { artifacts = await buildServer.GetArtifactsWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, cancellationToken); } } else { throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!"); } IEnumerable <BuildArtifact> pipelineArtifacts = artifacts.Where(a => string.Equals(a.Resource.Type, PipelineArtifactConstants.PipelineArtifact, StringComparison.OrdinalIgnoreCase)); if (pipelineArtifacts.Count() == 0) { throw new ArgumentException("Could not find any pipeline artifacts in the build."); } else { context.Output(StringUtil.Loc("DownloadingMultiplePipelineArtifacts", pipelineArtifacts.Count())); var artifactNameAndManifestIds = pipelineArtifacts.ToDictionary( keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data)); // 2) download to the target path var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds( artifactNameAndManifestIds, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters, minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName); PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) => new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { await dedupManifestClient.DownloadAsync(options, cancellationToken); }, maxRetries: 3, tracer: tracer, canRetryDelegate: e => true, context: nameof(DownloadAsync), cancellationToken: cancellationToken, continueOnCapturedContext: false); }); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord); } } else if (downloadOptions == DownloadOptions.SingleDownload) { // 1) get manifest id from artifact data BuildArtifact buildArtifact; if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId) { buildArtifact = await buildServer.GetArtifact(downloadParameters.ProjectId, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken); } else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName) { if (string.IsNullOrEmpty(downloadParameters.ProjectName)) { throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!"); } else { buildArtifact = await buildServer.GetArtifactWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken); } } else { throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!"); } var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data); var options = DownloadDedupManifestArtifactOptions.CreateWithManifestId( manifestId, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters); PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) => new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { await dedupManifestClient.DownloadAsync(options, cancellationToken); }, maxRetries: 3, tracer: tracer, canRetryDelegate: e => true, context: nameof(DownloadAsync), cancellationToken: cancellationToken, continueOnCapturedContext: false); }); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord); } else { throw new InvalidOperationException($"Invalid {nameof(downloadOptions)}!"); } } }
/// <summary> /// Alternative for UploadAndReferenceBlobAsync that uses WalkBlocksAsync instead of the synchronous WalkBlocks. /// Also utilizes the AsyncHttpRetryHelper to mitigate transient exceptions. /// </summary> public static Task UploadAndReferenceBlobWithRetriesAsync( this IBlobStoreHttpClient client, BlobIdentifier blobId, Stream stream, BlobReference reference, Context context, CancellationToken cts) { Contract.Requires(stream != null); var attempt = 0; return(AsyncHttpRetryHelper.InvokeVoidAsync( async() => { bool blobUploaded = false; stream.Position = 0; attempt++; await VsoHash.WalkBlocksAsync( stream, blockActionSemaphore: null, multiBlocksInParallel: true, singleBlockCallback: async(block, blockLength, blockHash) => { await client.PutSingleBlockBlobAndReferenceAsync( blobId, block, blockLength, reference, cts).ConfigureAwait(false); blobUploaded = true; }, multiBlockCallback: (block, blockLength, blockHash, isFinalBlock) => client.PutBlobBlockAsync(blobId, block, blockLength, cts), multiBlockSealCallback: async blobIdWithBlocks => { var failedRefs = await client.TryReferenceWithBlocksAsync( new Dictionary <BlobIdentifierWithBlocks, IEnumerable <BlobReference> > { { blobIdWithBlocks, new[] { reference } } }, cancellationToken: cts).ConfigureAwait(false); blobUploaded = !failedRefs.Any(); } ); if (!blobUploaded) { throw new AsyncHttpRetryHelper.RetryableException($"Could not upload blob on attempt {attempt}."); } }, maxRetries: 5, tracer: new AppTraceSourceContextAdapter(context, "BlobStoreHttpClient", SourceLevels.All), canRetryDelegate: exception => { // HACK HACK: This is an additional layer of retries specifically to catch the SSL exceptions seen in DM. // Newer versions of Artifact packages have this retry automatically, but the packages deployed with the M119.0 release do not. // Once the next release is completed, these retries can be removed. if (exception is HttpRequestException && exception.InnerException is WebException) { return true; } while (exception != null) { if (exception is TimeoutException) { return true; } exception = exception.InnerException; } return false; }, cancellationToken: cts, continueOnCapturedContext: false, context: context.Id.ToString())); }
private async Task DownloadFileContainerAsync(IEnumerable <FileContainerItem> items, ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true) { var containerIdAndRoot = ParseContainerId(artifact.Resource.Data); var projectId = downloadParameters.ProjectId; tracer.Info($"Start downloading FCS artifact- {artifact.Name}"); if (!isSingleArtifactDownload && items.Any()) { Directory.CreateDirectory(rootPath); } var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder); Parallel.ForEach(folderItems, (folder) => { var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); Directory.CreateDirectory(targetPath); }); var fileItems = items.Where(i => i.ItemType == ContainerItemType.File); // Only initialize these clients if we know we need to download from Blobstore // If a client cannot connect to Blobstore, we shouldn't stop them from downloading from FCS var downloadFromBlob = !AgentKnobs.DisableBuildArtifactsToBlob.GetValue(context).AsBoolean(); DedupStoreClient dedupClient = null; BlobStoreClientTelemetryTfs clientTelemetry = null; if (downloadFromBlob && fileItems.Any(x => x.BlobMetadata != null)) { try { (dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupClientAsync( false, (str) => this.tracer.Info(str), this.connection, DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context), cancellationToken); } catch (SocketException e) { ExceptionsUtil.HandleSocketException(e, connection.Uri.ToString(), context.Warning); } catch { var blobStoreHost = dedupClient.Client.BaseAddress.Host; var allowListLink = BlobStoreWarningInfoProvider.GetAllowListLinkForCurrentPlatform(); var warningMessage = StringUtil.Loc("BlobStoreDownloadWarning", blobStoreHost, allowListLink); // Fall back to streaming through TFS if we cannot reach blobstore downloadFromBlob = false; tracer.Warn(warningMessage); } } var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>( async item => { var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); var directory = Path.GetDirectoryName(targetPath); Directory.CreateDirectory(directory); await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { tracer.Info($"Downloading: {targetPath}"); if (item.BlobMetadata != null && downloadFromBlob) { await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, dedupClient, clientTelemetry, cancellationToken); } else { using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken)) using (var targetStream = new FileStream(targetPath, FileMode.Create)) { await sourceStream.CopyToAsync(targetStream); } } }, maxRetries: downloadParameters.RetryDownloadCount, cancellationToken: cancellationToken, tracer: tracer, continueOnCapturedContext: false, canRetryDelegate: exception => exception is IOException, context: null ); }, new ExecutionDataflowBlockOptions() { BoundedCapacity = 5000, MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit, CancellationToken = cancellationToken, }); await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken); // Send results to CustomerIntelligence if (clientTelemetry != null) { var planId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.PlanId)?.Value ?? Guid.Empty.ToString()); var jobId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.JobId)?.Value ?? Guid.Empty.ToString()); context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.BuildArtifactDownload, properties: clientTelemetry.GetArtifactDownloadTelemetry(planId, jobId)); } // check files (will throw an exception if a file is corrupt) if (downloadParameters.CheckDownloadedFiles) { CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); } }
private async Task DownloadFileContainerAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true) { var containerIdAndRoot = ParseContainerId(artifact.Resource.Data); var projectId = downloadParameters.ProjectId; var minimatchPatterns = downloadParameters.MinimatchFilters; var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, isShallow : false, includeBlobMetadata : true, containerIdAndRoot.Item2); tracer.Info($"Start downloading FCS artifact- {artifact.Name}"); IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer, downloadParameters.CustomMinimatchOptions); if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0) { items = this.GetFilteredItems(items, minimatcherFuncs); } if (!isSingleArtifactDownload && items.Any()) { Directory.CreateDirectory(rootPath); } var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder); Parallel.ForEach(folderItems, (folder) => { var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); Directory.CreateDirectory(targetPath); }); var fileItems = items.Where(i => i.ItemType == ContainerItemType.File); // Only initialize these clients if we know we need to download from Blobstore // If a client cannot connect to Blobstore, we shouldn't stop them from downloading from FCS var downloadFromBlob = !AgentKnobs.DisableBuildArtifactsToBlob.GetValue(context).AsBoolean(); DedupStoreClient dedupClient = null; BlobStoreClientTelemetryTfs clientTelemetry = null; if (downloadFromBlob && fileItems.Any(x => x.BlobMetadata != null)) { (dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupClientAsync( false, (str) => this.tracer.Info(str), this.connection, cancellationToken); } var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>( async item => { var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); var directory = Path.GetDirectoryName(targetPath); Directory.CreateDirectory(directory); await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { tracer.Info($"Downloading: {targetPath}"); if (item.BlobMetadata != null && downloadFromBlob) { await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, dedupClient, clientTelemetry, cancellationToken); } else { using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken)) using (var targetStream = new FileStream(targetPath, FileMode.Create)) { await sourceStream.CopyToAsync(targetStream); } } }, maxRetries: downloadParameters.RetryDownloadCount, cancellationToken: cancellationToken, tracer: tracer, continueOnCapturedContext: false, canRetryDelegate: exception => exception is IOException, context: null ); }, new ExecutionDataflowBlockOptions() { BoundedCapacity = 5000, MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit, CancellationToken = cancellationToken, }); await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken); // Send results to CustomerIntelligence if (clientTelemetry != null) { var planId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.PlanId)?.Value ?? Guid.Empty.ToString()); var jobId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.JobId)?.Value ?? Guid.Empty.ToString()); context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.BuildArtifactDownload, properties: clientTelemetry.GetArtifactDownloadTelemetry(planId, jobId)); } // check files (will throw an exception if a file is corrupt) if (downloadParameters.CheckDownloadedFiles) { CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); } }