// Returns all artifact items. Uses minimatch filters specified in downloadParameters.
private async Task <IEnumerable <FileContainerItem> > GetArtifactItems(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact)
{
(long, string)containerIdAndRoot = ParseContainerId(buildArtifact.Resource.Data);
Guid projectId = downloadParameters.ProjectId;
string[] minimatchPatterns = downloadParameters.MinimatchFilters;
List <FileContainerItem> items = await containerClient.QueryContainerItemsAsync(
containerIdAndRoot.Item1,
projectId,
isShallow : false,
includeBlobMetadata : true,
containerIdAndRoot.Item2
);
IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(
minimatchPatterns,
tracer,
downloadParameters.CustomMinimatchOptions
);
if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0)
{
items = this.GetFilteredItems(items, minimatcherFuncs);
}
return(items);
}
private async Task <ArtifactRecord> DownloadFileShareArtifactAsync(
string sourcePath,
string destPath,
int parallelCount,
CancellationToken cancellationToken,
IEnumerable <string> minimatchPatterns = null)
{
Stopwatch watch = Stopwatch.StartNew();
IEnumerable <Func <string, bool> > minimatchFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, this.tracer);
var trimChars = new[] { '\\', '/' };
sourcePath = sourcePath.TrimEnd(trimChars);
var artifactName = new DirectoryInfo(destPath).Name;
IEnumerable <FileInfo> files =
new DirectoryInfo(sourcePath).EnumerateFiles("*", SearchOption.AllDirectories);
var parallelism = new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = parallelCount,
BoundedCapacity = 2 * parallelCount,
CancellationToken = cancellationToken
};
var contentSize = 0;
var fileCount = 0;
var actionBlock = NonSwallowingActionBlock.Create <FileInfo>(
action: async file =>
{
if (minimatchFuncs == null || minimatchFuncs.Any(match => match(file.FullName)))
{
string tempPath = Path.Combine(destPath, Path.GetRelativePath(sourcePath, file.FullName));
context.Output(StringUtil.Loc("CopyFileToDestination", file, tempPath));
FileInfo tempFile = new System.IO.FileInfo(tempPath);
using (StreamReader fileReader = GetFileReader(file.FullName))
{
await WriteStreamToFile(
fileReader.BaseStream,
tempFile.FullName,
DefaultStreamBufferSize,
cancellationToken);
}
Interlocked.Add(ref contentSize, tempPath.Length);
Interlocked.Increment(ref fileCount);
}
},
dataflowBlockOptions: parallelism);
await actionBlock.SendAllAndCompleteAsync(files, actionBlock, cancellationToken);
watch.Stop();
return(new ArtifactRecord(artifactName,
fileCount,
contentSize,
watch.ElapsedMilliseconds));
}
private async Task DownloadFileContainerAsync(Guid projectId, BuildArtifact artifact, string rootPath, IEnumerable <string> minimatchPatterns, CancellationToken cancellationToken, bool isSingleArtifactDownload = true)
{
var containerIdAndRoot = ParseContainerId(artifact.Resource.Data);
var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, containerIdAndRoot.Item2);
tracer.Info($"Start downloading FCS artifact- {artifact.Name}");
IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer);
if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0)
{
items = this.GetFilteredItems(items, minimatcherFuncs);
}
if (!isSingleArtifactDownload && items.Any())
{
Directory.CreateDirectory(rootPath);
}
var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder);
Parallel.ForEach(folderItems, (folder) =>
{
var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2);
Directory.CreateDirectory(targetPath);
});
var fileItems = items.Where(i => i.ItemType == ContainerItemType.File);
var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>(
async item =>
{
var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2);
var directory = Path.GetDirectoryName(targetPath);
Directory.CreateDirectory(directory);
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken))
{
tracer.Info($"Downloading: {targetPath}");
using (var targetStream = new FileStream(targetPath, FileMode.Create))
{
await sourceStream.CopyToAsync(targetStream);
}
}
},
maxRetries: 5,
cancellationToken: cancellationToken,
tracer: tracer,
continueOnCapturedContext: false,
canRetryDelegate: exception => exception is IOException,
context: null
);
},
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 5000,
MaxDegreeOfParallelism = 8,
CancellationToken = cancellationToken,
});
await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken);
}
private async Task DownloadFileContainerAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true)
{
var containerIdAndRoot = ParseContainerId(artifact.Resource.Data);
var projectId = downloadParameters.ProjectId;
var minimatchPatterns = downloadParameters.MinimatchFilters;
var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, isShallow : false, includeBlobMetadata : true, containerIdAndRoot.Item2);
tracer.Info($"Start downloading FCS artifact- {artifact.Name}");
IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer);
if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0)
{
items = this.GetFilteredItems(items, minimatcherFuncs);
}
if (!isSingleArtifactDownload && items.Any())
{
Directory.CreateDirectory(rootPath);
}
var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder);
Parallel.ForEach(folderItems, (folder) =>
{
var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
Directory.CreateDirectory(targetPath);
});
var fileItems = items.Where(i => i.ItemType == ContainerItemType.File);
var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>(
async item =>
{
var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
var directory = Path.GetDirectoryName(targetPath);
Directory.CreateDirectory(directory);
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
tracer.Info($"Downloading: {targetPath}");
if (item.BlobMetadata != null)
{
await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, cancellationToken);
}
else
{
using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken))
using (var targetStream = new FileStream(targetPath, FileMode.Create))
{
await sourceStream.CopyToAsync(targetStream);
}
}
},
maxRetries: downloadParameters.RetryDownloadCount,
cancellationToken: cancellationToken,
tracer: tracer,
continueOnCapturedContext: false,
canRetryDelegate: exception => exception is IOException,
context: null
);
},
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 5000,
MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit,
CancellationToken = cancellationToken,
});
await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken);
// check files (will throw an exception if a file is corrupt)
if (downloadParameters.CheckDownloadedFiles)
{
CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
}
}
private async Task CopyFileShareAsync(string downloadRootPath, string destPath, IEnumerable <string> minimatchPatterns, CancellationToken cancellationToken)
{
IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, this.tracer);
await DownloadFileShareArtifactAsync(downloadRootPath, destPath, defaultParallelCount, cancellationToken, minimatcherFuncs);
}
/// <summary>
/// Collects hashtable with items in accordance with patterns
/// </summary>
/// <param name="paths">List of relative paths for items detected in artifact. The relative paths start from name of artifact.</param>
/// <param name="minimatchPatterns">Array of patterns used to filter items in artifact</param>
/// <param name="customMinimatchOptions">Download parameters from minimatcherFuncs</param>
/// <returns></returns>
public dynamic GetMapToFilterItems(List <string> paths, string[] minimatchPatterns, Options customMinimatchOptions)
{
// Hashtable to keep track of matches.
Hashtable map = new Hashtable();
foreach (string minimatchPattern in minimatchPatterns)
{
tracer.Info($"Pattern: {minimatchPattern}");
// Trim and skip empty.
string currentPattern = minimatchPattern.Trim();
if (String.IsNullOrEmpty(currentPattern))
{
tracer.Info($"Skipping empty pattern.");
continue;
}
// Clone match options.
Options matchOptions = CloneMiniMatchOptions(customMinimatchOptions);
// Skip comments.
if (!matchOptions.NoComment && currentPattern.StartsWith('#'))
{
tracer.Info($"Skipping comment.");
continue;
}
// Set NoComment. Brace expansion could result in a leading '#'.
matchOptions.NoComment = true;
// Determine whether pattern is include or exclude.
int negateCount = 0;
if (!matchOptions.NoNegate)
{
while (negateCount < currentPattern.Length && currentPattern[negateCount] == '!')
{
negateCount++;
}
currentPattern = currentPattern.Substring(negateCount);
if (negateCount > 0)
{
tracer.Info($"Trimmed leading '!'. Pattern: '{currentPattern}'");
}
}
bool isIncludePattern = negateCount == 0 ||
(negateCount % 2 == 0 && !matchOptions.FlipNegate) ||
(negateCount % 2 == 1 && matchOptions.FlipNegate);
// Set NoNegate. Brace expansion could result in a leading '!'.
matchOptions.NoNegate = true;
matchOptions.FlipNegate = false;
// Trim and skip empty.
currentPattern = currentPattern.Trim();
if (String.IsNullOrEmpty(currentPattern))
{
tracer.Info($"Skipping empty pattern.");
continue;
}
// Expand braces - required to accurately interpret findPath.
string[] expandedPatterns;
string preExpandedPattern = currentPattern;
if (matchOptions.NoBrace)
{
expandedPatterns = new string[] { currentPattern };
}
else
{
expandedPatterns = ExpandBraces(currentPattern, matchOptions);
}
// Set NoBrace.
matchOptions.NoBrace = true;
foreach (string expandedPattern in expandedPatterns)
{
if (expandedPattern != preExpandedPattern)
{
tracer.Info($"Pattern: {expandedPattern}");
}
// Trim and skip empty.
currentPattern = expandedPattern.Trim();
if (String.IsNullOrEmpty(currentPattern))
{
tracer.Info($"Skipping empty pattern.");
continue;
}
string[] currentPatterns = new string[] { currentPattern };
IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(
currentPatterns,
tracer,
matchOptions
);
UpdatePatternsMap(isIncludePattern, paths, minimatcherFuncs, ref map);
}
}
return(map);
}
private async Task DownloadFileContainerAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true)
{
var containerIdAndRoot = ParseContainerId(artifact.Resource.Data);
var projectId = downloadParameters.ProjectId;
var minimatchPatterns = downloadParameters.MinimatchFilters;
var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, isShallow : false, includeBlobMetadata : true, containerIdAndRoot.Item2);
tracer.Info($"Start downloading FCS artifact- {artifact.Name}");
IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer, downloadParameters.CustomMinimatchOptions);
if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0)
{
items = this.GetFilteredItems(items, minimatcherFuncs);
}
if (!isSingleArtifactDownload && items.Any())
{
Directory.CreateDirectory(rootPath);
}
var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder);
Parallel.ForEach(folderItems, (folder) =>
{
var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
Directory.CreateDirectory(targetPath);
});
var fileItems = items.Where(i => i.ItemType == ContainerItemType.File);
// Only initialize these clients if we know we need to download from Blobstore
// If a client cannot connect to Blobstore, we shouldn't stop them from downloading from FCS
var downloadFromBlob = !AgentKnobs.DisableBuildArtifactsToBlob.GetValue(context).AsBoolean();
DedupStoreClient dedupClient = null;
BlobStoreClientTelemetryTfs clientTelemetry = null;
if (downloadFromBlob && fileItems.Any(x => x.BlobMetadata != null))
{
(dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupClientAsync(
false, (str) => this.tracer.Info(str), this.connection, cancellationToken);
}
var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>(
async item =>
{
var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
var directory = Path.GetDirectoryName(targetPath);
Directory.CreateDirectory(directory);
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
tracer.Info($"Downloading: {targetPath}");
if (item.BlobMetadata != null && downloadFromBlob)
{
await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, dedupClient, clientTelemetry, cancellationToken);
}
else
{
using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken))
using (var targetStream = new FileStream(targetPath, FileMode.Create))
{
await sourceStream.CopyToAsync(targetStream);
}
}
},
maxRetries: downloadParameters.RetryDownloadCount,
cancellationToken: cancellationToken,
tracer: tracer,
continueOnCapturedContext: false,
canRetryDelegate: exception => exception is IOException,
context: null
);
},
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 5000,
MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit,
CancellationToken = cancellationToken,
});
await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken);
// Send results to CustomerIntelligence
if (clientTelemetry != null)
{
var planId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.PlanId)?.Value ?? Guid.Empty.ToString());
var jobId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.JobId)?.Value ?? Guid.Empty.ToString());
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.BuildArtifactDownload,
properties: clientTelemetry.GetArtifactDownloadTelemetry(planId, jobId));
}
// check files (will throw an exception if a file is corrupt)
if (downloadParameters.CheckDownloadedFiles)
{
CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
}
}
private async Task <ArtifactRecord> DownloadFileShareArtifactAsync(
string sourcePath,
string destPath,
int parallelCount,
ArtifactDownloadParameters downloadParameters,
CancellationToken cancellationToken,
IEnumerable <string> minimatchPatterns = null)
{
Stopwatch watch = Stopwatch.StartNew();
IEnumerable <Func <string, bool> > minimatchFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, this.tracer);
var trimChars = new[] { '\\', '/' };
sourcePath = sourcePath.TrimEnd(trimChars);
var artifactName = new DirectoryInfo(sourcePath).Name;
List <FileInfo> files =
new DirectoryInfo(sourcePath).EnumerateFiles("*", SearchOption.AllDirectories).ToList <FileInfo>();
ArtifactItemFilters filters = new ArtifactItemFilters(connection, tracer);
// Getting list of file paths. It is useful to handle list of paths instead of files.
// Also it allows to use the same methods for FileContainerProvider and FileShareProvider.
List <string> paths = new List <string>();
foreach (FileInfo file in files)
{
string pathInArtifact = filters.RemoveSourceDirFromPath(file, sourcePath);
paths.Add(Path.Combine(artifactName, pathInArtifact));
}
Options customMinimatchOptions;
if (downloadParameters.CustomMinimatchOptions != null)
{
customMinimatchOptions = downloadParameters.CustomMinimatchOptions;
}
else
{
customMinimatchOptions = new Options()
{
Dot = true,
NoBrace = true,
AllowWindowsPaths = PlatformUtil.RunningOnWindows
};
}
Hashtable map = filters.GetMapToFilterItems(paths, downloadParameters.MinimatchFilters, customMinimatchOptions);
// Returns filtered list of artifact items. Uses minimatch filters specified in downloadParameters.
IEnumerable <FileInfo> filteredFiles = filters.ApplyPatternsMapToFileShareItems(files, map, sourcePath);
tracer.Info($"{filteredFiles.ToList().Count} final results");
IEnumerable <FileInfo> excludedItems = files.Except(filteredFiles);
foreach (FileInfo item in excludedItems)
{
tracer.Info($"File excluded: {item.FullName}");
}
var parallelism = new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = parallelCount,
BoundedCapacity = 2 * parallelCount,
CancellationToken = cancellationToken
};
var contentSize = 0;
var fileCount = 0;
var actionBlock = NonSwallowingActionBlock.Create <FileInfo>(
action: async file =>
{
if (minimatchFuncs == null || minimatchFuncs.Any(match => match(file.FullName)))
{
string tempPath = Path.Combine(destPath, Path.GetRelativePath(sourcePath, file.FullName));
context.Output(StringUtil.Loc("CopyFileToDestination", file, tempPath));
FileInfo tempFile = new System.IO.FileInfo(tempPath);
using (StreamReader fileReader = GetFileReader(file.FullName))
{
await WriteStreamToFile(
fileReader.BaseStream,
tempFile.FullName,
DefaultStreamBufferSize,
cancellationToken);
}
Interlocked.Add(ref contentSize, tempPath.Length);
Interlocked.Increment(ref fileCount);
}
},
dataflowBlockOptions: parallelism);
await actionBlock.SendAllAndCompleteAsync(filteredFiles, actionBlock, cancellationToken);
watch.Stop();
return(new ArtifactRecord(artifactName,
fileCount,
contentSize,
watch.ElapsedMilliseconds));
}
