// Returns all artifact items. Uses minimatch filters specified in downloadParameters. private async Task <IEnumerable <FileContainerItem> > GetArtifactItems(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact) { (long, string)containerIdAndRoot = ParseContainerId(buildArtifact.Resource.Data); Guid projectId = downloadParameters.ProjectId; string[] minimatchPatterns = downloadParameters.MinimatchFilters; List <FileContainerItem> items = await containerClient.QueryContainerItemsAsync( containerIdAndRoot.Item1, projectId, isShallow : false, includeBlobMetadata : true, containerIdAndRoot.Item2 ); IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs( minimatchPatterns, tracer, downloadParameters.CustomMinimatchOptions ); if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0) { items = this.GetFilteredItems(items, minimatcherFuncs); } return(items); }
private async Task <ArtifactRecord> DownloadFileShareArtifactAsync( string sourcePath, string destPath, int parallelCount, CancellationToken cancellationToken, IEnumerable <string> minimatchPatterns = null) { Stopwatch watch = Stopwatch.StartNew(); IEnumerable <Func <string, bool> > minimatchFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, this.tracer); var trimChars = new[] { '\\', '/' }; sourcePath = sourcePath.TrimEnd(trimChars); var artifactName = new DirectoryInfo(destPath).Name; IEnumerable <FileInfo> files = new DirectoryInfo(sourcePath).EnumerateFiles("*", SearchOption.AllDirectories); var parallelism = new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = parallelCount, BoundedCapacity = 2 * parallelCount, CancellationToken = cancellationToken }; var contentSize = 0; var fileCount = 0; var actionBlock = NonSwallowingActionBlock.Create <FileInfo>( action: async file => { if (minimatchFuncs == null || minimatchFuncs.Any(match => match(file.FullName))) { string tempPath = Path.Combine(destPath, Path.GetRelativePath(sourcePath, file.FullName)); context.Output(StringUtil.Loc("CopyFileToDestination", file, tempPath)); FileInfo tempFile = new System.IO.FileInfo(tempPath); using (StreamReader fileReader = GetFileReader(file.FullName)) { await WriteStreamToFile( fileReader.BaseStream, tempFile.FullName, DefaultStreamBufferSize, cancellationToken); } Interlocked.Add(ref contentSize, tempPath.Length); Interlocked.Increment(ref fileCount); } }, dataflowBlockOptions: parallelism); await actionBlock.SendAllAndCompleteAsync(files, actionBlock, cancellationToken); watch.Stop(); return(new ArtifactRecord(artifactName, fileCount, contentSize, watch.ElapsedMilliseconds)); }
private async Task DownloadFileContainerAsync(Guid projectId, BuildArtifact artifact, string rootPath, IEnumerable <string> minimatchPatterns, CancellationToken cancellationToken, bool isSingleArtifactDownload = true) { var containerIdAndRoot = ParseContainerId(artifact.Resource.Data); var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, containerIdAndRoot.Item2); tracer.Info($"Start downloading FCS artifact- {artifact.Name}"); IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer); if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0) { items = this.GetFilteredItems(items, minimatcherFuncs); } if (!isSingleArtifactDownload && items.Any()) { Directory.CreateDirectory(rootPath); } var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder); Parallel.ForEach(folderItems, (folder) => { var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2); Directory.CreateDirectory(targetPath); }); var fileItems = items.Where(i => i.ItemType == ContainerItemType.File); var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>( async item => { var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2); var directory = Path.GetDirectoryName(targetPath); Directory.CreateDirectory(directory); await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken)) { tracer.Info($"Downloading: {targetPath}"); using (var targetStream = new FileStream(targetPath, FileMode.Create)) { await sourceStream.CopyToAsync(targetStream); } } }, maxRetries: 5, cancellationToken: cancellationToken, tracer: tracer, continueOnCapturedContext: false, canRetryDelegate: exception => exception is IOException, context: null ); }, new ExecutionDataflowBlockOptions() { BoundedCapacity = 5000, MaxDegreeOfParallelism = 8, CancellationToken = cancellationToken, }); await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken); }
private async Task DownloadFileContainerAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true) { var containerIdAndRoot = ParseContainerId(artifact.Resource.Data); var projectId = downloadParameters.ProjectId; var minimatchPatterns = downloadParameters.MinimatchFilters; var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, isShallow : false, includeBlobMetadata : true, containerIdAndRoot.Item2); tracer.Info($"Start downloading FCS artifact- {artifact.Name}"); IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer); if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0) { items = this.GetFilteredItems(items, minimatcherFuncs); } if (!isSingleArtifactDownload && items.Any()) { Directory.CreateDirectory(rootPath); } var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder); Parallel.ForEach(folderItems, (folder) => { var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); Directory.CreateDirectory(targetPath); }); var fileItems = items.Where(i => i.ItemType == ContainerItemType.File); var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>( async item => { var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); var directory = Path.GetDirectoryName(targetPath); Directory.CreateDirectory(directory); await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { tracer.Info($"Downloading: {targetPath}"); if (item.BlobMetadata != null) { await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, cancellationToken); } else { using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken)) using (var targetStream = new FileStream(targetPath, FileMode.Create)) { await sourceStream.CopyToAsync(targetStream); } } }, maxRetries: downloadParameters.RetryDownloadCount, cancellationToken: cancellationToken, tracer: tracer, continueOnCapturedContext: false, canRetryDelegate: exception => exception is IOException, context: null ); }, new ExecutionDataflowBlockOptions() { BoundedCapacity = 5000, MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit, CancellationToken = cancellationToken, }); await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken); // check files (will throw an exception if a file is corrupt) if (downloadParameters.CheckDownloadedFiles) { CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); } }
private async Task CopyFileShareAsync(string downloadRootPath, string destPath, IEnumerable <string> minimatchPatterns, CancellationToken cancellationToken) { IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, this.tracer); await DownloadFileShareArtifactAsync(downloadRootPath, destPath, defaultParallelCount, cancellationToken, minimatcherFuncs); }
/// <summary> /// Collects hashtable with items in accordance with patterns /// </summary> /// <param name="paths">List of relative paths for items detected in artifact. The relative paths start from name of artifact.</param> /// <param name="minimatchPatterns">Array of patterns used to filter items in artifact</param> /// <param name="customMinimatchOptions">Download parameters from minimatcherFuncs</param> /// <returns></returns> public dynamic GetMapToFilterItems(List <string> paths, string[] minimatchPatterns, Options customMinimatchOptions) { // Hashtable to keep track of matches. Hashtable map = new Hashtable(); foreach (string minimatchPattern in minimatchPatterns) { tracer.Info($"Pattern: {minimatchPattern}"); // Trim and skip empty. string currentPattern = minimatchPattern.Trim(); if (String.IsNullOrEmpty(currentPattern)) { tracer.Info($"Skipping empty pattern."); continue; } // Clone match options. Options matchOptions = CloneMiniMatchOptions(customMinimatchOptions); // Skip comments. if (!matchOptions.NoComment && currentPattern.StartsWith('#')) { tracer.Info($"Skipping comment."); continue; } // Set NoComment. Brace expansion could result in a leading '#'. matchOptions.NoComment = true; // Determine whether pattern is include or exclude. int negateCount = 0; if (!matchOptions.NoNegate) { while (negateCount < currentPattern.Length && currentPattern[negateCount] == '!') { negateCount++; } currentPattern = currentPattern.Substring(negateCount); if (negateCount > 0) { tracer.Info($"Trimmed leading '!'. Pattern: '{currentPattern}'"); } } bool isIncludePattern = negateCount == 0 || (negateCount % 2 == 0 && !matchOptions.FlipNegate) || (negateCount % 2 == 1 && matchOptions.FlipNegate); // Set NoNegate. Brace expansion could result in a leading '!'. matchOptions.NoNegate = true; matchOptions.FlipNegate = false; // Trim and skip empty. currentPattern = currentPattern.Trim(); if (String.IsNullOrEmpty(currentPattern)) { tracer.Info($"Skipping empty pattern."); continue; } // Expand braces - required to accurately interpret findPath. string[] expandedPatterns; string preExpandedPattern = currentPattern; if (matchOptions.NoBrace) { expandedPatterns = new string[] { currentPattern }; } else { expandedPatterns = ExpandBraces(currentPattern, matchOptions); } // Set NoBrace. matchOptions.NoBrace = true; foreach (string expandedPattern in expandedPatterns) { if (expandedPattern != preExpandedPattern) { tracer.Info($"Pattern: {expandedPattern}"); } // Trim and skip empty. currentPattern = expandedPattern.Trim(); if (String.IsNullOrEmpty(currentPattern)) { tracer.Info($"Skipping empty pattern."); continue; } string[] currentPatterns = new string[] { currentPattern }; IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs( currentPatterns, tracer, matchOptions ); UpdatePatternsMap(isIncludePattern, paths, minimatcherFuncs, ref map); } } return(map); }
private async Task DownloadFileContainerAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true) { var containerIdAndRoot = ParseContainerId(artifact.Resource.Data); var projectId = downloadParameters.ProjectId; var minimatchPatterns = downloadParameters.MinimatchFilters; var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, isShallow : false, includeBlobMetadata : true, containerIdAndRoot.Item2); tracer.Info($"Start downloading FCS artifact- {artifact.Name}"); IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer, downloadParameters.CustomMinimatchOptions); if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0) { items = this.GetFilteredItems(items, minimatcherFuncs); } if (!isSingleArtifactDownload && items.Any()) { Directory.CreateDirectory(rootPath); } var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder); Parallel.ForEach(folderItems, (folder) => { var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); Directory.CreateDirectory(targetPath); }); var fileItems = items.Where(i => i.ItemType == ContainerItemType.File); // Only initialize these clients if we know we need to download from Blobstore // If a client cannot connect to Blobstore, we shouldn't stop them from downloading from FCS var downloadFromBlob = !AgentKnobs.DisableBuildArtifactsToBlob.GetValue(context).AsBoolean(); DedupStoreClient dedupClient = null; BlobStoreClientTelemetryTfs clientTelemetry = null; if (downloadFromBlob && fileItems.Any(x => x.BlobMetadata != null)) { (dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupClientAsync( false, (str) => this.tracer.Info(str), this.connection, cancellationToken); } var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>( async item => { var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); var directory = Path.GetDirectoryName(targetPath); Directory.CreateDirectory(directory); await AsyncHttpRetryHelper.InvokeVoidAsync( async() => { tracer.Info($"Downloading: {targetPath}"); if (item.BlobMetadata != null && downloadFromBlob) { await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, dedupClient, clientTelemetry, cancellationToken); } else { using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken)) using (var targetStream = new FileStream(targetPath, FileMode.Create)) { await sourceStream.CopyToAsync(targetStream); } } }, maxRetries: downloadParameters.RetryDownloadCount, cancellationToken: cancellationToken, tracer: tracer, continueOnCapturedContext: false, canRetryDelegate: exception => exception is IOException, context: null ); }, new ExecutionDataflowBlockOptions() { BoundedCapacity = 5000, MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit, CancellationToken = cancellationToken, }); await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken); // Send results to CustomerIntelligence if (clientTelemetry != null) { var planId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.PlanId)?.Value ?? Guid.Empty.ToString()); var jobId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.JobId)?.Value ?? Guid.Empty.ToString()); context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.BuildArtifactDownload, properties: clientTelemetry.GetArtifactDownloadTelemetry(planId, jobId)); } // check files (will throw an exception if a file is corrupt) if (downloadParameters.CheckDownloadedFiles) { CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath); } }
private async Task <ArtifactRecord> DownloadFileShareArtifactAsync( string sourcePath, string destPath, int parallelCount, ArtifactDownloadParameters downloadParameters, CancellationToken cancellationToken, IEnumerable <string> minimatchPatterns = null) { Stopwatch watch = Stopwatch.StartNew(); IEnumerable <Func <string, bool> > minimatchFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, this.tracer); var trimChars = new[] { '\\', '/' }; sourcePath = sourcePath.TrimEnd(trimChars); var artifactName = new DirectoryInfo(sourcePath).Name; List <FileInfo> files = new DirectoryInfo(sourcePath).EnumerateFiles("*", SearchOption.AllDirectories).ToList <FileInfo>(); ArtifactItemFilters filters = new ArtifactItemFilters(connection, tracer); // Getting list of file paths. It is useful to handle list of paths instead of files. // Also it allows to use the same methods for FileContainerProvider and FileShareProvider. List <string> paths = new List <string>(); foreach (FileInfo file in files) { string pathInArtifact = filters.RemoveSourceDirFromPath(file, sourcePath); paths.Add(Path.Combine(artifactName, pathInArtifact)); } Options customMinimatchOptions; if (downloadParameters.CustomMinimatchOptions != null) { customMinimatchOptions = downloadParameters.CustomMinimatchOptions; } else { customMinimatchOptions = new Options() { Dot = true, NoBrace = true, AllowWindowsPaths = PlatformUtil.RunningOnWindows }; } Hashtable map = filters.GetMapToFilterItems(paths, downloadParameters.MinimatchFilters, customMinimatchOptions); // Returns filtered list of artifact items. Uses minimatch filters specified in downloadParameters. IEnumerable <FileInfo> filteredFiles = filters.ApplyPatternsMapToFileShareItems(files, map, sourcePath); tracer.Info($"{filteredFiles.ToList().Count} final results"); IEnumerable <FileInfo> excludedItems = files.Except(filteredFiles); foreach (FileInfo item in excludedItems) { tracer.Info($"File excluded: {item.FullName}"); } var parallelism = new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = parallelCount, BoundedCapacity = 2 * parallelCount, CancellationToken = cancellationToken }; var contentSize = 0; var fileCount = 0; var actionBlock = NonSwallowingActionBlock.Create <FileInfo>( action: async file => { if (minimatchFuncs == null || minimatchFuncs.Any(match => match(file.FullName))) { string tempPath = Path.Combine(destPath, Path.GetRelativePath(sourcePath, file.FullName)); context.Output(StringUtil.Loc("CopyFileToDestination", file, tempPath)); FileInfo tempFile = new System.IO.FileInfo(tempPath); using (StreamReader fileReader = GetFileReader(file.FullName)) { await WriteStreamToFile( fileReader.BaseStream, tempFile.FullName, DefaultStreamBufferSize, cancellationToken); } Interlocked.Add(ref contentSize, tempPath.Length); Interlocked.Increment(ref fileCount); } }, dataflowBlockOptions: parallelism); await actionBlock.SendAllAndCompleteAsync(filteredFiles, actionBlock, cancellationToken); watch.Stop(); return(new ArtifactRecord(artifactName, fileCount, contentSize, watch.ElapsedMilliseconds)); }