// Returns list of filtered artifact items. Uses minimatch filters specified in downloadParameters. private async Task <IEnumerable <FileContainerItem> > GetArtifactItems(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact) { (long, string)containerIdAndRoot = ParseContainerId(buildArtifact.Resource.Data); Guid projectId = downloadParameters.ProjectId; string[] minimatchPatterns = downloadParameters.MinimatchFilters; List <FileContainerItem> items = await containerClient.QueryContainerItemsAsync( containerIdAndRoot.Item1, projectId, isShallow : false, includeBlobMetadata : true, containerIdAndRoot.Item2 ); Options customMinimatchOptions; if (downloadParameters.CustomMinimatchOptions != null) { customMinimatchOptions = downloadParameters.CustomMinimatchOptions; } else { customMinimatchOptions = new Options() { Dot = true, NoBrace = true, AllowWindowsPaths = PlatformUtil.RunningOnWindows }; } // Getting list of item paths. It is useful to handle list of paths instead of items. // Also it allows to use the same methods for FileContainerProvider and FileShareProvider. List <string> paths = new List <string>(); foreach (FileContainerItem item in items) { paths.Add(item.Path); } ArtifactItemFilters filters = new ArtifactItemFilters(connection, tracer); Hashtable map = filters.GetMapToFilterItems(paths, minimatchPatterns, customMinimatchOptions); // Returns filtered list of artifact items. Uses minimatch filters specified in downloadParameters. List <FileContainerItem> resultItems = filters.ApplyPatternsMapToContainerItems(items, map); tracer.Info($"{resultItems.Count} final results"); IEnumerable <FileContainerItem> excludedItems = items.Except(resultItems); foreach (FileContainerItem item in excludedItems) { tracer.Info($"Item excluded: {item.Path}"); } return(resultItems); }
private async Task <ArtifactRecord> DownloadFileShareArtifactAsync( string sourcePath, string destPath, int parallelCount, ArtifactDownloadParameters downloadParameters, CancellationToken cancellationToken, IEnumerable <string> minimatchPatterns = null) { Stopwatch watch = Stopwatch.StartNew(); IEnumerable <Func <string, bool> > minimatchFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, this.tracer); var trimChars = new[] { '\\', '/' }; sourcePath = sourcePath.TrimEnd(trimChars); var artifactName = new DirectoryInfo(sourcePath).Name; List <FileInfo> files = new DirectoryInfo(sourcePath).EnumerateFiles("*", SearchOption.AllDirectories).ToList <FileInfo>(); ArtifactItemFilters filters = new ArtifactItemFilters(connection, tracer); // Getting list of file paths. It is useful to handle list of paths instead of files. // Also it allows to use the same methods for FileContainerProvider and FileShareProvider. List <string> paths = new List <string>(); foreach (FileInfo file in files) { string pathInArtifact = filters.RemoveSourceDirFromPath(file, sourcePath); paths.Add(Path.Combine(artifactName, pathInArtifact)); } Options customMinimatchOptions; if (downloadParameters.CustomMinimatchOptions != null) { customMinimatchOptions = downloadParameters.CustomMinimatchOptions; } else { customMinimatchOptions = new Options() { Dot = true, NoBrace = true, AllowWindowsPaths = PlatformUtil.RunningOnWindows }; } Hashtable map = filters.GetMapToFilterItems(paths, downloadParameters.MinimatchFilters, customMinimatchOptions); // Returns filtered list of artifact items. Uses minimatch filters specified in downloadParameters. IEnumerable <FileInfo> filteredFiles = filters.ApplyPatternsMapToFileShareItems(files, map, sourcePath); tracer.Info($"{filteredFiles.ToList().Count} final results"); IEnumerable <FileInfo> excludedItems = files.Except(filteredFiles); foreach (FileInfo item in excludedItems) { tracer.Info($"File excluded: {item.FullName}"); } var parallelism = new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = parallelCount, BoundedCapacity = 2 * parallelCount, CancellationToken = cancellationToken }; var contentSize = 0; var fileCount = 0; var actionBlock = NonSwallowingActionBlock.Create <FileInfo>( action: async file => { if (minimatchFuncs == null || minimatchFuncs.Any(match => match(file.FullName))) { string tempPath = Path.Combine(destPath, Path.GetRelativePath(sourcePath, file.FullName)); context.Output(StringUtil.Loc("CopyFileToDestination", file, tempPath)); FileInfo tempFile = new System.IO.FileInfo(tempPath); using (StreamReader fileReader = GetFileReader(file.FullName)) { await WriteStreamToFile( fileReader.BaseStream, tempFile.FullName, DefaultStreamBufferSize, cancellationToken); } Interlocked.Add(ref contentSize, tempPath.Length); Interlocked.Increment(ref fileCount); } }, dataflowBlockOptions: parallelism); await actionBlock.SendAllAndCompleteAsync(filteredFiles, actionBlock, cancellationToken); watch.Stop(); return(new ArtifactRecord(artifactName, fileCount, contentSize, watch.ElapsedMilliseconds)); }