// Returns all artifact items. Uses minimatch filters specified in downloadParameters. private async Task <IEnumerable <FileContainerItem> > GetArtifactItems(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact) { (long, string)containerIdAndRoot = ParseContainerId(buildArtifact.Resource.Data); Guid projectId = downloadParameters.ProjectId; string[] minimatchPatterns = downloadParameters.MinimatchFilters; List <FileContainerItem> items = await containerClient.QueryContainerItemsAsync( containerIdAndRoot.Item1, projectId, isShallow : false, includeBlobMetadata : true, containerIdAndRoot.Item2 ); IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs( minimatchPatterns, tracer, downloadParameters.CustomMinimatchOptions ); if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0) { items = this.GetFilteredItems(items, minimatcherFuncs); } return(items); }
public async Task DownloadMultipleArtifactsAsync(ArtifactDownloadParameters downloadParameters, IEnumerable <BuildArtifact> buildArtifacts, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context) { var allFileArtifactPaths = new List <string>(); foreach (var buildArtifact in buildArtifacts) { var dirPath = downloadParameters.AppendArtifactNameToTargetPath ? Path.Combine(downloadParameters.TargetDirectory, buildArtifact.Name) : downloadParameters.TargetDirectory; IEnumerable <FileContainerItem> items = await GetArtifactItems(downloadParameters, buildArtifact); IEnumerable <string> fileArtifactPaths = items .Where((item) => item.ItemType == ContainerItemType.File) .Select((fileItem) => Path.Combine(dirPath, fileItem.Path)); allFileArtifactPaths.AddRange(fileArtifactPaths); await DownloadFileContainerAsync(items, downloadParameters, buildArtifact, dirPath, context, cancellationToken, isSingleArtifactDownload : false); } if (downloadParameters.ExtractTars) { ExtractTarsIfPresent(context, allFileArtifactPaths, downloadParameters.TargetDirectory, downloadParameters.ExtractedTarsTempPath); } }
public async Task DownloadMultipleArtifactsAsync(ArtifactDownloadParameters downloadParameters, IEnumerable <BuildArtifact> buildArtifacts, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context) { foreach (var buildArtifact in buildArtifacts) { var dirPath = Path.Combine(downloadParameters.TargetDirectory, buildArtifact.Name); await DownloadFileContainerAsync(downloadParameters, buildArtifact, dirPath, context, cancellationToken, isSingleArtifactDownload : false); } }
// Returns list of filtered artifact items. Uses minimatch filters specified in downloadParameters. private async Task <IEnumerable <FileContainerItem> > GetArtifactItems(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact) { (long, string)containerIdAndRoot = ParseContainerId(buildArtifact.Resource.Data); Guid projectId = downloadParameters.ProjectId; string[] minimatchPatterns = downloadParameters.MinimatchFilters; List <FileContainerItem> items = await containerClient.QueryContainerItemsAsync( containerIdAndRoot.Item1, projectId, isShallow : false, includeBlobMetadata : true, containerIdAndRoot.Item2 ); Options customMinimatchOptions; if (downloadParameters.CustomMinimatchOptions != null) { customMinimatchOptions = downloadParameters.CustomMinimatchOptions; } else { customMinimatchOptions = new Options() { Dot = true, NoBrace = true, AllowWindowsPaths = PlatformUtil.RunningOnWindows }; } // Getting list of item paths. It is useful to handle list of paths instead of items. // Also it allows to use the same methods for FileContainerProvider and FileShareProvider. List <string> paths = new List <string>(); foreach (FileContainerItem item in items) { paths.Add(item.Path); } ArtifactItemFilters filters = new ArtifactItemFilters(connection, tracer); Hashtable map = filters.GetMapToFilterItems(paths, minimatchPatterns, customMinimatchOptions); // Returns filtered list of artifact items. Uses minimatch filters specified in downloadParameters. List <FileContainerItem> resultItems = filters.ApplyPatternsMapToContainerItems(items, map); tracer.Info($"{resultItems.Count} final results"); IEnumerable <FileContainerItem> excludedItems = items.Except(resultItems); foreach (FileContainerItem item in excludedItems) { tracer.Info($"Item excluded: {item.Path}"); } return(resultItems); }
public async Task DownloadSingleArtifactAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context) { IEnumerable <FileContainerItem> items = await GetArtifactItems(downloadParameters, buildArtifact); await this.DownloadFileContainerAsync(items, downloadParameters, buildArtifact, downloadParameters.TargetDirectory, context, cancellationToken); IEnumerable <string> fileArtifactPaths = items .Where((item) => item.ItemType == ContainerItemType.File) .Select((fileItem) => Path.Combine(downloadParameters.TargetDirectory, fileItem.Path)); if (downloadParameters.ExtractTars) { ExtractTarsIfPresent(context, fileArtifactPaths, downloadParameters.TargetDirectory, downloadParameters.ExtractedTarsTempPath); } }
private async Task <FileShareDownloadResult> DownloadArtifactsAsync(ArtifactDownloadParameters downloadParameters, IEnumerable <BuildArtifact> buildArtifacts, CancellationToken cancellationToken) { var records = new List <ArtifactRecord>(); long totalContentSize = 0; int totalFileCount = 0; foreach (var buildArtifact in buildArtifacts) { var downloadRootPath = Path.Combine(buildArtifact.Resource.Data, buildArtifact.Name); var minimatchPatterns = downloadParameters.MinimatchFilters.Select(pattern => Path.Combine(buildArtifact.Resource.Data, pattern)); var record = await this.DownloadFileShareArtifactAsync(downloadRootPath, Path.Combine(downloadParameters.TargetDirectory, buildArtifact.Name), defaultParallelCount, cancellationToken, minimatchPatterns); totalContentSize += record.ContentSize; totalFileCount += record.FileCount; records.Add(record); } return(new FileShareDownloadResult(records, totalFileCount, totalContentSize)); }
// Download pipeline artifact from Azure DevOps BlobStore service through DedupManifestArtifactClient to a target path // Old V0 function internal Task DownloadAsync( AgentTaskPluginExecutionContext context, Guid projectId, int pipelineId, string artifactName, string targetDir, CancellationToken cancellationToken) { var downloadParameters = new ArtifactDownloadParameters { ProjectRetrievalOptions = BuildArtifactRetrievalOptions.RetrieveByProjectId, ProjectId = projectId, PipelineId = pipelineId, ArtifactName = artifactName, TargetDirectory = targetDir }; return(this.DownloadAsync(context, downloadParameters, DownloadOptions.SingleDownload, cancellationToken)); }
public async Task DownloadMultipleArtifactsAsync(ArtifactDownloadParameters downloadParameters, IEnumerable <BuildArtifact> buildArtifacts, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context) { var(dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClientAsync( this.context.IsSystemDebugTrue(), (str) => this.context.Output(str), this.connection, cancellationToken); using (clientTelemetry) { var artifactNameAndManifestIds = buildArtifacts.ToDictionary( keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data)); // 2) download to the target path var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds( artifactNameAndManifestIds, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters, minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName); PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) => new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { await dedupManifestClient.DownloadAsync(options, cancellationToken); }, maxRetries: 3, tracer: tracer, canRetryDelegate: e => true, context: nameof(DownloadMultipleArtifactsAsync), cancellationToken: cancellationToken, continueOnCapturedContext: false); }); // Send results to CustomerIntelligence this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord); } }
public async Task DownloadMultipleArtifactsAsync(ArtifactDownloadParameters downloadParameters, IEnumerable <BuildArtifact> buildArtifacts, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context) { context.Warning(StringUtil.Loc("DownloadArtifactWarning", "UNC")); var(dedupManifestClient, clientTelemetry) = await this.factory.CreateDedupManifestClientAsync(context.IsSystemDebugTrue(), (str) => context.Output(str), connection, cancellationToken); using (clientTelemetry) { FileShareActionRecord downloadRecord = clientTelemetry.CreateRecord <FileShareActionRecord>((level, uri, type) => new FileShareActionRecord(level, uri, type, nameof(DownloadArtifactsAsync), context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { return(await DownloadArtifactsAsync(downloadParameters, buildArtifacts, cancellationToken)); } ); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord); } }
public async Task DownloadSingleArtifactAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context) { var(dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClientAsync( this.context.IsSystemDebugTrue(), (str) => this.context.Output(str), this.connection, cancellationToken); using (clientTelemetry) { var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data); var options = DownloadDedupManifestArtifactOptions.CreateWithManifestId( manifestId, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters); PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) => new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { await dedupManifestClient.DownloadAsync(options, cancellationToken); }, maxRetries: 3, tracer: tracer, canRetryDelegate: e => true, context: nameof(DownloadSingleArtifactAsync), cancellationToken: cancellationToken, continueOnCapturedContext: false); }); // Send results to CustomerIntelligence this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord); } }
private async Task <ArtifactRecord> DownloadFileShareArtifactAsync( string sourcePath, string destPath, int parallelCount, ArtifactDownloadParameters downloadParameters, CancellationToken cancellationToken, IEnumerable <string> minimatchPatterns = null) { Stopwatch watch = Stopwatch.StartNew(); IEnumerable <Func <string, bool> > minimatchFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, this.tracer); var trimChars = new[] { '\\', '/' }; sourcePath = sourcePath.TrimEnd(trimChars); var artifactName = new DirectoryInfo(sourcePath).Name; List <FileInfo> files = new DirectoryInfo(sourcePath).EnumerateFiles("*", SearchOption.AllDirectories).ToList <FileInfo>(); ArtifactItemFilters filters = new ArtifactItemFilters(connection, tracer); // Getting list of file paths. It is useful to handle list of paths instead of files. // Also it allows to use the same methods for FileContainerProvider and FileShareProvider. List <string> paths = new List <string>(); foreach (FileInfo file in files) { string pathInArtifact = filters.RemoveSourceDirFromPath(file, sourcePath); paths.Add(Path.Combine(artifactName, pathInArtifact)); } Options customMinimatchOptions; if (downloadParameters.CustomMinimatchOptions != null) { customMinimatchOptions = downloadParameters.CustomMinimatchOptions; } else { customMinimatchOptions = new Options() { Dot = true, NoBrace = true, AllowWindowsPaths = PlatformUtil.RunningOnWindows }; } Hashtable map = filters.GetMapToFilterItems(paths, downloadParameters.MinimatchFilters, customMinimatchOptions); // Returns filtered list of artifact items. Uses minimatch filters specified in downloadParameters. IEnumerable <FileInfo> filteredFiles = filters.ApplyPatternsMapToFileShareItems(files, map, sourcePath); tracer.Info($"{filteredFiles.ToList().Count} final results"); IEnumerable <FileInfo> excludedItems = files.Except(filteredFiles); foreach (FileInfo item in excludedItems) { tracer.Info($"File excluded: {item.FullName}"); } var parallelism = new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = parallelCount, BoundedCapacity = 2 * parallelCount, CancellationToken = cancellationToken }; var contentSize = 0; var fileCount = 0; var actionBlock = NonSwallowingActionBlock.Create <FileInfo>( action: async file => { if (minimatchFuncs == null || minimatchFuncs.Any(match => match(file.FullName))) { string tempPath = Path.Combine(destPath, Path.GetRelativePath(sourcePath, file.FullName)); context.Output(StringUtil.Loc("CopyFileToDestination", file, tempPath)); FileInfo tempFile = new System.IO.FileInfo(tempPath); using (StreamReader fileReader = GetFileReader(file.FullName)) { await WriteStreamToFile( fileReader.BaseStream, tempFile.FullName, DefaultStreamBufferSize, cancellationToken); } Interlocked.Add(ref contentSize, tempPath.Length); Interlocked.Increment(ref fileCount); } }, dataflowBlockOptions: parallelism); await actionBlock.SendAllAndCompleteAsync(filteredFiles, actionBlock, cancellationToken); watch.Stop(); return(new ArtifactRecord(artifactName, fileCount, contentSize, watch.ElapsedMilliseconds)); }
private async Task DownloadFileContainerAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true) { var containerIdAndRoot = ParseContainerId(artifact.Resource.Data); var projectId = downloadParameters.ProjectId; var minimatchPatterns = downloadParameters.MinimatchFilters; var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, isShallow : false, includeBlobMetadata : true, containerIdAndRoot.Item2); tracer.Info($"Start downloading FCS artifact- {artifact.Name}"); IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer); if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0) { items = this.GetFilteredItems(items, minimatcherFuncs); } if (!isSingleArtifactDownload && items.Any()) { Directory.CreateDirectory(rootPath); } var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder); Parallel.ForEach(folderItems, (folder) => { var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); Directory.CreateDirectory(targetPath); }); var fileItems = items.Where(i => i.ItemType == ContainerItemType.File); var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>( async item => { var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); var directory = Path.GetDirectoryName(targetPath); Directory.CreateDirectory(directory); await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { tracer.Info($"Downloading: {targetPath}"); if (item.BlobMetadata != null) { await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, cancellationToken); } else { using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken)) using (var targetStream = new FileStream(targetPath, FileMode.Create)) { await sourceStream.CopyToAsync(targetStream); } } }, maxRetries: downloadParameters.RetryDownloadCount, cancellationToken: cancellationToken, tracer: tracer, continueOnCapturedContext: false, canRetryDelegate: exception => exception is IOException, context: null ); }, new ExecutionDataflowBlockOptions() { BoundedCapacity = 5000, MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit, CancellationToken = cancellationToken, }); await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken); // check files (will throw an exception if a file is corrupt) if (downloadParameters.CheckDownloadedFiles) { CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); } }
private async Task DownloadFileContainerAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true) { var containerIdAndRoot = ParseContainerId(artifact.Resource.Data); var projectId = downloadParameters.ProjectId; var minimatchPatterns = downloadParameters.MinimatchFilters; var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, isShallow : false, includeBlobMetadata : true, containerIdAndRoot.Item2); tracer.Info($"Start downloading FCS artifact- {artifact.Name}"); IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer, downloadParameters.CustomMinimatchOptions); if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0) { items = this.GetFilteredItems(items, minimatcherFuncs); } if (!isSingleArtifactDownload && items.Any()) { Directory.CreateDirectory(rootPath); } var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder); Parallel.ForEach(folderItems, (folder) => { var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); Directory.CreateDirectory(targetPath); }); var fileItems = items.Where(i => i.ItemType == ContainerItemType.File); // Only initialize these clients if we know we need to download from Blobstore // If a client cannot connect to Blobstore, we shouldn't stop them from downloading from FCS var downloadFromBlob = !AgentKnobs.DisableBuildArtifactsToBlob.GetValue(context).AsBoolean(); DedupStoreClient dedupClient = null; BlobStoreClientTelemetryTfs clientTelemetry = null; if (downloadFromBlob && fileItems.Any(x => x.BlobMetadata != null)) { (dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupClientAsync( false, (str) => this.tracer.Info(str), this.connection, cancellationToken); } var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>( async item => { var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); var directory = Path.GetDirectoryName(targetPath); Directory.CreateDirectory(directory); await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { tracer.Info($"Downloading: {targetPath}"); if (item.BlobMetadata != null && downloadFromBlob) { await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, dedupClient, clientTelemetry, cancellationToken); } else { using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken)) using (var targetStream = new FileStream(targetPath, FileMode.Create)) { await sourceStream.CopyToAsync(targetStream); } } }, maxRetries: downloadParameters.RetryDownloadCount, cancellationToken: cancellationToken, tracer: tracer, continueOnCapturedContext: false, canRetryDelegate: exception => exception is IOException, context: null ); }, new ExecutionDataflowBlockOptions() { BoundedCapacity = 5000, MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit, CancellationToken = cancellationToken, }); await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken); // Send results to CustomerIntelligence if (clientTelemetry != null) { var planId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.PlanId)?.Value ?? Guid.Empty.ToString()); var jobId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.JobId)?.Value ?? Guid.Empty.ToString()); context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.BuildArtifactDownload, properties: clientTelemetry.GetArtifactDownloadTelemetry(planId, jobId)); } // check files (will throw an exception if a file is corrupt) if (downloadParameters.CheckDownloadedFiles) { CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); } }
public async Task DownloadSingleArtifactAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context) { await this.DownloadFileContainerAsync(downloadParameters, buildArtifact, downloadParameters.TargetDirectory, context, cancellationToken); }
// Download for version 2. This decision was made because version 1 is sealed and we didn't want to break any existing customers. internal async Task DownloadAsyncV2( AgentTaskPluginExecutionContext context, ArtifactDownloadParameters downloadParameters, DownloadOptions downloadOptions, CancellationToken cancellationToken) { VssConnection connection = context.VssConnection; BuildServer buildServer = new BuildServer(connection); // download all pipeline artifacts if artifact name is missing if (downloadOptions == DownloadOptions.MultiDownload) { List <BuildArtifact> artifacts; if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId) { artifacts = await buildServer.GetArtifactsAsync(downloadParameters.ProjectId, downloadParameters.PipelineId, cancellationToken); } else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName) { if (string.IsNullOrEmpty(downloadParameters.ProjectName)) { throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!"); } else { artifacts = await buildServer.GetArtifactsWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, cancellationToken); } } else { throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!"); } IEnumerable <BuildArtifact> buildArtifacts = artifacts.Where(a => string.Equals(a.Resource.Type, PipelineArtifactConstants.Container, StringComparison.OrdinalIgnoreCase)); IEnumerable <BuildArtifact> pipelineArtifacts = artifacts.Where(a => string.Equals(a.Resource.Type, PipelineArtifactConstants.PipelineArtifact, StringComparison.OrdinalIgnoreCase)); IEnumerable <BuildArtifact> fileShareArtifacts = artifacts.Where(a => string.Equals(a.Resource.Type, PipelineArtifactConstants.FileShareArtifact, StringComparison.OrdinalIgnoreCase)); if (buildArtifacts.Any()) { FileContainerProvider provider = new FileContainerProvider(connection, this.tracer); await provider.DownloadMultipleArtifactsAsync(downloadParameters, buildArtifacts, cancellationToken, context); } if (pipelineArtifacts.Any()) { PipelineArtifactProvider provider = new PipelineArtifactProvider(context, connection, this.tracer); await provider.DownloadMultipleArtifactsAsync(downloadParameters, pipelineArtifacts, cancellationToken, context); } if (fileShareArtifacts.Any()) { FileShareProvider provider = new FileShareProvider(context, connection, this.tracer); await provider.DownloadMultipleArtifactsAsync(downloadParameters, fileShareArtifacts, cancellationToken, context); } } else if (downloadOptions == DownloadOptions.SingleDownload) { // 1) get manifest id from artifact data BuildArtifact buildArtifact; if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId) { buildArtifact = await buildServer.GetArtifact(downloadParameters.ProjectId, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken); } else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName) { if (string.IsNullOrEmpty(downloadParameters.ProjectName)) { throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!"); } else { buildArtifact = await buildServer.GetArtifactWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken); } } else { throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!"); } ArtifactProviderFactory factory = new ArtifactProviderFactory(context, connection, this.tracer); IArtifactProvider provider = factory.GetProvider(buildArtifact); await provider.DownloadSingleArtifactAsync(downloadParameters, buildArtifact, cancellationToken, context); } else { throw new InvalidOperationException($"Invalid {nameof(downloadOptions)}!"); } }
// Download with minimatch patterns, V1. internal async Task DownloadAsync( AgentTaskPluginExecutionContext context, ArtifactDownloadParameters downloadParameters, DownloadOptions downloadOptions, CancellationToken cancellationToken) { VssConnection connection = context.VssConnection; var(dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance .CreateDedupManifestClientAsync(context.IsSystemDebugTrue(), (str) => context.Output(str), connection, cancellationToken); BuildServer buildServer = new BuildServer(connection); using (clientTelemetry) { // download all pipeline artifacts if artifact name is missing if (downloadOptions == DownloadOptions.MultiDownload) { List <BuildArtifact> artifacts; if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId) { artifacts = await buildServer.GetArtifactsAsync(downloadParameters.ProjectId, downloadParameters.PipelineId, cancellationToken); } else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName) { if (string.IsNullOrEmpty(downloadParameters.ProjectName)) { throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!"); } else { artifacts = await buildServer.GetArtifactsWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, cancellationToken); } } else { throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!"); } IEnumerable <BuildArtifact> pipelineArtifacts = artifacts.Where(a => string.Equals(a.Resource.Type, PipelineArtifactConstants.PipelineArtifact, StringComparison.OrdinalIgnoreCase)); if (pipelineArtifacts.Count() == 0) { throw new ArgumentException("Could not find any pipeline artifacts in the build."); } else { context.Output(StringUtil.Loc("DownloadingMultiplePipelineArtifacts", pipelineArtifacts.Count())); var artifactNameAndManifestIds = pipelineArtifacts.ToDictionary( keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data)); // 2) download to the target path var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds( artifactNameAndManifestIds, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters, minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName, customMinimatchOptions: downloadParameters.CustomMinimatchOptions); PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) => new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { await dedupManifestClient.DownloadAsync(options, cancellationToken); }, maxRetries: 3, tracer: tracer, canRetryDelegate: e => true, context: nameof(DownloadAsync), cancellationToken: cancellationToken, continueOnCapturedContext: false); }); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord); } } else if (downloadOptions == DownloadOptions.SingleDownload) { // 1) get manifest id from artifact data BuildArtifact buildArtifact; if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId) { buildArtifact = await buildServer.GetArtifact(downloadParameters.ProjectId, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken); } else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName) { if (string.IsNullOrEmpty(downloadParameters.ProjectName)) { throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!"); } else { buildArtifact = await buildServer.GetArtifactWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken); } } else { throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!"); } var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data); var options = DownloadDedupManifestArtifactOptions.CreateWithManifestId( manifestId, downloadParameters.TargetDirectory, proxyUri: null, minimatchPatterns: downloadParameters.MinimatchFilters, customMinimatchOptions: downloadParameters.CustomMinimatchOptions); PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) => new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { await dedupManifestClient.DownloadAsync(options, cancellationToken); }, maxRetries: 3, tracer: tracer, canRetryDelegate: e => true, context: nameof(DownloadAsync), cancellationToken: cancellationToken, continueOnCapturedContext: false); }); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord); } else { throw new InvalidOperationException($"Invalid {nameof(downloadOptions)}!"); } } }
public async Task DownloadSingleArtifactAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context) { await DownloadMultipleArtifactsAsync(downloadParameters, new List <BuildArtifact> { buildArtifact }, cancellationToken, context); }
private async Task DownloadFileContainerAsync(IEnumerable <FileContainerItem> items, ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true) { var containerIdAndRoot = ParseContainerId(artifact.Resource.Data); var projectId = downloadParameters.ProjectId; tracer.Info($"Start downloading FCS artifact- {artifact.Name}"); if (!isSingleArtifactDownload && items.Any()) { Directory.CreateDirectory(rootPath); } var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder); Parallel.ForEach(folderItems, (folder) => { var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); Directory.CreateDirectory(targetPath); }); var fileItems = items.Where(i => i.ItemType == ContainerItemType.File); // Only initialize these clients if we know we need to download from Blobstore // If a client cannot connect to Blobstore, we shouldn't stop them from downloading from FCS var downloadFromBlob = !AgentKnobs.DisableBuildArtifactsToBlob.GetValue(context).AsBoolean(); DedupStoreClient dedupClient = null; BlobStoreClientTelemetryTfs clientTelemetry = null; if (downloadFromBlob && fileItems.Any(x => x.BlobMetadata != null)) { try { (dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupClientAsync( false, (str) => this.tracer.Info(str), this.connection, DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context), cancellationToken); } catch (SocketException e) { ExceptionsUtil.HandleSocketException(e, connection.Uri.ToString(), context.Warning); } catch { var blobStoreHost = dedupClient.Client.BaseAddress.Host; var allowListLink = BlobStoreWarningInfoProvider.GetAllowListLinkForCurrentPlatform(); var warningMessage = StringUtil.Loc("BlobStoreDownloadWarning", blobStoreHost, allowListLink); // Fall back to streaming through TFS if we cannot reach blobstore downloadFromBlob = false; tracer.Warn(warningMessage); } } var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>( async item => { var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); var directory = Path.GetDirectoryName(targetPath); Directory.CreateDirectory(directory); await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { tracer.Info($"Downloading: {targetPath}"); if (item.BlobMetadata != null && downloadFromBlob) { await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, dedupClient, clientTelemetry, cancellationToken); } else { using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken)) using (var targetStream = new FileStream(targetPath, FileMode.Create)) { await sourceStream.CopyToAsync(targetStream); } } }, maxRetries: downloadParameters.RetryDownloadCount, cancellationToken: cancellationToken, tracer: tracer, continueOnCapturedContext: false, canRetryDelegate: exception => exception is IOException, context: null ); }, new ExecutionDataflowBlockOptions() { BoundedCapacity = 5000, MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit, CancellationToken = cancellationToken, }); await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken); // Send results to CustomerIntelligence if (clientTelemetry != null) { var planId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.PlanId)?.Value ?? Guid.Empty.ToString()); var jobId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.JobId)?.Value ?? Guid.Empty.ToString()); context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.BuildArtifactDownload, properties: clientTelemetry.GetArtifactDownloadTelemetry(planId, jobId)); } // check files (will throw an exception if a file is corrupt) if (downloadParameters.CheckDownloadedFiles) { CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); } }