// Returns list of filtered artifact items. Uses minimatch filters specified in downloadParameters.
private async Task <IEnumerable <FileContainerItem> > GetArtifactItems(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact)
{
(long, string)containerIdAndRoot = ParseContainerId(buildArtifact.Resource.Data);
Guid projectId = downloadParameters.ProjectId;
string[] minimatchPatterns = downloadParameters.MinimatchFilters;
List <FileContainerItem> items = await containerClient.QueryContainerItemsAsync(
containerIdAndRoot.Item1,
projectId,
isShallow : false,
includeBlobMetadata : true,
containerIdAndRoot.Item2
);
Options customMinimatchOptions;
if (downloadParameters.CustomMinimatchOptions != null)
{
customMinimatchOptions = downloadParameters.CustomMinimatchOptions;
}
else
{
customMinimatchOptions = new Options()
{
Dot = true,
NoBrace = true,
AllowWindowsPaths = PlatformUtil.RunningOnWindows
};
}
// Getting list of item paths. It is useful to handle list of paths instead of items.
// Also it allows to use the same methods for FileContainerProvider and FileShareProvider.
List <string> paths = new List <string>();
foreach (FileContainerItem item in items)
{
paths.Add(item.Path);
}
ArtifactItemFilters filters = new ArtifactItemFilters(connection, tracer);
Hashtable map = filters.GetMapToFilterItems(paths, minimatchPatterns, customMinimatchOptions);
// Returns filtered list of artifact items. Uses minimatch filters specified in downloadParameters.
List <FileContainerItem> resultItems = filters.ApplyPatternsMapToContainerItems(items, map);
tracer.Info($"{resultItems.Count} final results");
IEnumerable <FileContainerItem> excludedItems = items.Except(resultItems);
foreach (FileContainerItem item in excludedItems)
{
tracer.Info($"Item excluded: {item.Path}");
}
return(resultItems);
}
private async Task <ArtifactRecord> DownloadFileShareArtifactAsync(
string sourcePath,
string destPath,
int parallelCount,
ArtifactDownloadParameters downloadParameters,
CancellationToken cancellationToken,
IEnumerable <string> minimatchPatterns = null)
{
Stopwatch watch = Stopwatch.StartNew();
IEnumerable <Func <string, bool> > minimatchFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, this.tracer);
var trimChars = new[] { '\\', '/' };
sourcePath = sourcePath.TrimEnd(trimChars);
var artifactName = new DirectoryInfo(sourcePath).Name;
List <FileInfo> files =
new DirectoryInfo(sourcePath).EnumerateFiles("*", SearchOption.AllDirectories).ToList <FileInfo>();
ArtifactItemFilters filters = new ArtifactItemFilters(connection, tracer);
// Getting list of file paths. It is useful to handle list of paths instead of files.
// Also it allows to use the same methods for FileContainerProvider and FileShareProvider.
List <string> paths = new List <string>();
foreach (FileInfo file in files)
{
string pathInArtifact = filters.RemoveSourceDirFromPath(file, sourcePath);
paths.Add(Path.Combine(artifactName, pathInArtifact));
}
Options customMinimatchOptions;
if (downloadParameters.CustomMinimatchOptions != null)
{
customMinimatchOptions = downloadParameters.CustomMinimatchOptions;
}
else
{
customMinimatchOptions = new Options()
{
Dot = true,
NoBrace = true,
AllowWindowsPaths = PlatformUtil.RunningOnWindows
};
}
Hashtable map = filters.GetMapToFilterItems(paths, downloadParameters.MinimatchFilters, customMinimatchOptions);
// Returns filtered list of artifact items. Uses minimatch filters specified in downloadParameters.
IEnumerable <FileInfo> filteredFiles = filters.ApplyPatternsMapToFileShareItems(files, map, sourcePath);
tracer.Info($"{filteredFiles.ToList().Count} final results");
IEnumerable <FileInfo> excludedItems = files.Except(filteredFiles);
foreach (FileInfo item in excludedItems)
{
tracer.Info($"File excluded: {item.FullName}");
}
var parallelism = new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = parallelCount,
BoundedCapacity = 2 * parallelCount,
CancellationToken = cancellationToken
};
var contentSize = 0;
var fileCount = 0;
var actionBlock = NonSwallowingActionBlock.Create <FileInfo>(
action: async file =>
{
if (minimatchFuncs == null || minimatchFuncs.Any(match => match(file.FullName)))
{
string tempPath = Path.Combine(destPath, Path.GetRelativePath(sourcePath, file.FullName));
context.Output(StringUtil.Loc("CopyFileToDestination", file, tempPath));
FileInfo tempFile = new System.IO.FileInfo(tempPath);
using (StreamReader fileReader = GetFileReader(file.FullName))
{
await WriteStreamToFile(
fileReader.BaseStream,
tempFile.FullName,
DefaultStreamBufferSize,
cancellationToken);
}
Interlocked.Add(ref contentSize, tempPath.Length);
Interlocked.Increment(ref fileCount);
}
},
dataflowBlockOptions: parallelism);
await actionBlock.SendAllAndCompleteAsync(filteredFiles, actionBlock, cancellationToken);
watch.Stop();
return(new ArtifactRecord(artifactName,
fileCount,
contentSize,
watch.ElapsedMilliseconds));
}
