// Returns all artifact items. Uses minimatch filters specified in downloadParameters.
private async Task <IEnumerable <FileContainerItem> > GetArtifactItems(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact)
{
(long, string)containerIdAndRoot = ParseContainerId(buildArtifact.Resource.Data);
Guid projectId = downloadParameters.ProjectId;
string[] minimatchPatterns = downloadParameters.MinimatchFilters;
List <FileContainerItem> items = await containerClient.QueryContainerItemsAsync(
containerIdAndRoot.Item1,
projectId,
isShallow : false,
includeBlobMetadata : true,
containerIdAndRoot.Item2
);
IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(
minimatchPatterns,
tracer,
downloadParameters.CustomMinimatchOptions
);
if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0)
{
items = this.GetFilteredItems(items, minimatcherFuncs);
}
return(items);
}
public async Task DownloadMultipleArtifactsAsync(ArtifactDownloadParameters downloadParameters, IEnumerable <BuildArtifact> buildArtifacts, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context)
{
var allFileArtifactPaths = new List <string>();
foreach (var buildArtifact in buildArtifacts)
{
var dirPath = downloadParameters.AppendArtifactNameToTargetPath
? Path.Combine(downloadParameters.TargetDirectory, buildArtifact.Name)
: downloadParameters.TargetDirectory;
IEnumerable <FileContainerItem> items = await GetArtifactItems(downloadParameters, buildArtifact);
IEnumerable <string> fileArtifactPaths = items
.Where((item) => item.ItemType == ContainerItemType.File)
.Select((fileItem) => Path.Combine(dirPath, fileItem.Path));
allFileArtifactPaths.AddRange(fileArtifactPaths);
await DownloadFileContainerAsync(items, downloadParameters, buildArtifact, dirPath, context, cancellationToken, isSingleArtifactDownload : false);
}
if (downloadParameters.ExtractTars)
{
ExtractTarsIfPresent(context, allFileArtifactPaths, downloadParameters.TargetDirectory, downloadParameters.ExtractedTarsTempPath);
}
}
public async Task DownloadMultipleArtifactsAsync(ArtifactDownloadParameters downloadParameters, IEnumerable <BuildArtifact> buildArtifacts, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context)
{
foreach (var buildArtifact in buildArtifacts)
{
var dirPath = Path.Combine(downloadParameters.TargetDirectory, buildArtifact.Name);
await DownloadFileContainerAsync(downloadParameters, buildArtifact, dirPath, context, cancellationToken, isSingleArtifactDownload : false);
}
}
// Returns list of filtered artifact items. Uses minimatch filters specified in downloadParameters.
private async Task <IEnumerable <FileContainerItem> > GetArtifactItems(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact)
{
(long, string)containerIdAndRoot = ParseContainerId(buildArtifact.Resource.Data);
Guid projectId = downloadParameters.ProjectId;
string[] minimatchPatterns = downloadParameters.MinimatchFilters;
List <FileContainerItem> items = await containerClient.QueryContainerItemsAsync(
containerIdAndRoot.Item1,
projectId,
isShallow : false,
includeBlobMetadata : true,
containerIdAndRoot.Item2
);
Options customMinimatchOptions;
if (downloadParameters.CustomMinimatchOptions != null)
{
customMinimatchOptions = downloadParameters.CustomMinimatchOptions;
}
else
{
customMinimatchOptions = new Options()
{
Dot = true,
NoBrace = true,
AllowWindowsPaths = PlatformUtil.RunningOnWindows
};
}
// Getting list of item paths. It is useful to handle list of paths instead of items.
// Also it allows to use the same methods for FileContainerProvider and FileShareProvider.
List <string> paths = new List <string>();
foreach (FileContainerItem item in items)
{
paths.Add(item.Path);
}
ArtifactItemFilters filters = new ArtifactItemFilters(connection, tracer);
Hashtable map = filters.GetMapToFilterItems(paths, minimatchPatterns, customMinimatchOptions);
// Returns filtered list of artifact items. Uses minimatch filters specified in downloadParameters.
List <FileContainerItem> resultItems = filters.ApplyPatternsMapToContainerItems(items, map);
tracer.Info($"{resultItems.Count} final results");
IEnumerable <FileContainerItem> excludedItems = items.Except(resultItems);
foreach (FileContainerItem item in excludedItems)
{
tracer.Info($"Item excluded: {item.Path}");
}
return(resultItems);
}
public async Task DownloadSingleArtifactAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context)
{
IEnumerable <FileContainerItem> items = await GetArtifactItems(downloadParameters, buildArtifact);
await this.DownloadFileContainerAsync(items, downloadParameters, buildArtifact, downloadParameters.TargetDirectory, context, cancellationToken);
IEnumerable <string> fileArtifactPaths = items
.Where((item) => item.ItemType == ContainerItemType.File)
.Select((fileItem) => Path.Combine(downloadParameters.TargetDirectory, fileItem.Path));
if (downloadParameters.ExtractTars)
{
ExtractTarsIfPresent(context, fileArtifactPaths, downloadParameters.TargetDirectory, downloadParameters.ExtractedTarsTempPath);
}
}
private async Task <FileShareDownloadResult> DownloadArtifactsAsync(ArtifactDownloadParameters downloadParameters, IEnumerable <BuildArtifact> buildArtifacts, CancellationToken cancellationToken)
{
var records = new List <ArtifactRecord>();
long totalContentSize = 0;
int totalFileCount = 0;
foreach (var buildArtifact in buildArtifacts)
{
var downloadRootPath = Path.Combine(buildArtifact.Resource.Data, buildArtifact.Name);
var minimatchPatterns = downloadParameters.MinimatchFilters.Select(pattern => Path.Combine(buildArtifact.Resource.Data, pattern));
var record = await this.DownloadFileShareArtifactAsync(downloadRootPath, Path.Combine(downloadParameters.TargetDirectory, buildArtifact.Name), defaultParallelCount, cancellationToken, minimatchPatterns);
totalContentSize += record.ContentSize;
totalFileCount += record.FileCount;
records.Add(record);
}
return(new FileShareDownloadResult(records, totalFileCount, totalContentSize));
}
// Download pipeline artifact from Azure DevOps BlobStore service through DedupManifestArtifactClient to a target path
// Old V0 function
internal Task DownloadAsync(
AgentTaskPluginExecutionContext context,
Guid projectId,
int pipelineId,
string artifactName,
string targetDir,
CancellationToken cancellationToken)
{
var downloadParameters = new ArtifactDownloadParameters
{
ProjectRetrievalOptions = BuildArtifactRetrievalOptions.RetrieveByProjectId,
ProjectId = projectId,
PipelineId = pipelineId,
ArtifactName = artifactName,
TargetDirectory = targetDir
};
return(this.DownloadAsync(context, downloadParameters, DownloadOptions.SingleDownload, cancellationToken));
}
public async Task DownloadMultipleArtifactsAsync(ArtifactDownloadParameters downloadParameters, IEnumerable <BuildArtifact> buildArtifacts, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context)
{
var(dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClientAsync(
this.context.IsSystemDebugTrue(), (str) => this.context.Output(str), this.connection, cancellationToken);
using (clientTelemetry) {
var artifactNameAndManifestIds = buildArtifacts.ToDictionary(
keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong
elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data));
// 2) download to the target path
var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds(
artifactNameAndManifestIds,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters,
minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName);
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true,
context: nameof(DownloadMultipleArtifactsAsync),
cancellationToken: cancellationToken,
continueOnCapturedContext: false);
});
// Send results to CustomerIntelligence
this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
}
public async Task DownloadMultipleArtifactsAsync(ArtifactDownloadParameters downloadParameters, IEnumerable <BuildArtifact> buildArtifacts, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context)
{
context.Warning(StringUtil.Loc("DownloadArtifactWarning", "UNC"));
var(dedupManifestClient, clientTelemetry) = await this.factory.CreateDedupManifestClientAsync(context.IsSystemDebugTrue(), (str) => context.Output(str), connection, cancellationToken);
using (clientTelemetry)
{
FileShareActionRecord downloadRecord = clientTelemetry.CreateRecord <FileShareActionRecord>((level, uri, type) =>
new FileShareActionRecord(level, uri, type, nameof(DownloadArtifactsAsync), context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
return(await DownloadArtifactsAsync(downloadParameters, buildArtifacts, cancellationToken));
}
);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
}
public async Task DownloadSingleArtifactAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context)
{
var(dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClientAsync(
this.context.IsSystemDebugTrue(), (str) => this.context.Output(str), this.connection, cancellationToken);
using (clientTelemetry) {
var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data);
var options = DownloadDedupManifestArtifactOptions.CreateWithManifestId(
manifestId,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters);
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true,
context: nameof(DownloadSingleArtifactAsync),
cancellationToken: cancellationToken,
continueOnCapturedContext: false);
});
// Send results to CustomerIntelligence
this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
}
private async Task <ArtifactRecord> DownloadFileShareArtifactAsync(
string sourcePath,
string destPath,
int parallelCount,
ArtifactDownloadParameters downloadParameters,
CancellationToken cancellationToken,
IEnumerable <string> minimatchPatterns = null)
{
Stopwatch watch = Stopwatch.StartNew();
IEnumerable <Func <string, bool> > minimatchFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, this.tracer);
var trimChars = new[] { '\\', '/' };
sourcePath = sourcePath.TrimEnd(trimChars);
var artifactName = new DirectoryInfo(sourcePath).Name;
List <FileInfo> files =
new DirectoryInfo(sourcePath).EnumerateFiles("*", SearchOption.AllDirectories).ToList <FileInfo>();
ArtifactItemFilters filters = new ArtifactItemFilters(connection, tracer);
// Getting list of file paths. It is useful to handle list of paths instead of files.
// Also it allows to use the same methods for FileContainerProvider and FileShareProvider.
List <string> paths = new List <string>();
foreach (FileInfo file in files)
{
string pathInArtifact = filters.RemoveSourceDirFromPath(file, sourcePath);
paths.Add(Path.Combine(artifactName, pathInArtifact));
}
Options customMinimatchOptions;
if (downloadParameters.CustomMinimatchOptions != null)
{
customMinimatchOptions = downloadParameters.CustomMinimatchOptions;
}
else
{
customMinimatchOptions = new Options()
{
Dot = true,
NoBrace = true,
AllowWindowsPaths = PlatformUtil.RunningOnWindows
};
}
Hashtable map = filters.GetMapToFilterItems(paths, downloadParameters.MinimatchFilters, customMinimatchOptions);
// Returns filtered list of artifact items. Uses minimatch filters specified in downloadParameters.
IEnumerable <FileInfo> filteredFiles = filters.ApplyPatternsMapToFileShareItems(files, map, sourcePath);
tracer.Info($"{filteredFiles.ToList().Count} final results");
IEnumerable <FileInfo> excludedItems = files.Except(filteredFiles);
foreach (FileInfo item in excludedItems)
{
tracer.Info($"File excluded: {item.FullName}");
}
var parallelism = new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = parallelCount,
BoundedCapacity = 2 * parallelCount,
CancellationToken = cancellationToken
};
var contentSize = 0;
var fileCount = 0;
var actionBlock = NonSwallowingActionBlock.Create <FileInfo>(
action: async file =>
{
if (minimatchFuncs == null || minimatchFuncs.Any(match => match(file.FullName)))
{
string tempPath = Path.Combine(destPath, Path.GetRelativePath(sourcePath, file.FullName));
context.Output(StringUtil.Loc("CopyFileToDestination", file, tempPath));
FileInfo tempFile = new System.IO.FileInfo(tempPath);
using (StreamReader fileReader = GetFileReader(file.FullName))
{
await WriteStreamToFile(
fileReader.BaseStream,
tempFile.FullName,
DefaultStreamBufferSize,
cancellationToken);
}
Interlocked.Add(ref contentSize, tempPath.Length);
Interlocked.Increment(ref fileCount);
}
},
dataflowBlockOptions: parallelism);
await actionBlock.SendAllAndCompleteAsync(filteredFiles, actionBlock, cancellationToken);
watch.Stop();
return(new ArtifactRecord(artifactName,
fileCount,
contentSize,
watch.ElapsedMilliseconds));
}
private async Task DownloadFileContainerAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true)
{
var containerIdAndRoot = ParseContainerId(artifact.Resource.Data);
var projectId = downloadParameters.ProjectId;
var minimatchPatterns = downloadParameters.MinimatchFilters;
var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, isShallow : false, includeBlobMetadata : true, containerIdAndRoot.Item2);
tracer.Info($"Start downloading FCS artifact- {artifact.Name}");
IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer);
if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0)
{
items = this.GetFilteredItems(items, minimatcherFuncs);
}
if (!isSingleArtifactDownload && items.Any())
{
Directory.CreateDirectory(rootPath);
}
var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder);
Parallel.ForEach(folderItems, (folder) =>
{
var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
Directory.CreateDirectory(targetPath);
});
var fileItems = items.Where(i => i.ItemType == ContainerItemType.File);
var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>(
async item =>
{
var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
var directory = Path.GetDirectoryName(targetPath);
Directory.CreateDirectory(directory);
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
tracer.Info($"Downloading: {targetPath}");
if (item.BlobMetadata != null)
{
await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, cancellationToken);
}
else
{
using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken))
using (var targetStream = new FileStream(targetPath, FileMode.Create))
{
await sourceStream.CopyToAsync(targetStream);
}
}
},
maxRetries: downloadParameters.RetryDownloadCount,
cancellationToken: cancellationToken,
tracer: tracer,
continueOnCapturedContext: false,
canRetryDelegate: exception => exception is IOException,
context: null
);
},
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 5000,
MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit,
CancellationToken = cancellationToken,
});
await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken);
// check files (will throw an exception if a file is corrupt)
if (downloadParameters.CheckDownloadedFiles)
{
CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
}
}
private async Task DownloadFileContainerAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true)
{
var containerIdAndRoot = ParseContainerId(artifact.Resource.Data);
var projectId = downloadParameters.ProjectId;
var minimatchPatterns = downloadParameters.MinimatchFilters;
var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, isShallow : false, includeBlobMetadata : true, containerIdAndRoot.Item2);
tracer.Info($"Start downloading FCS artifact- {artifact.Name}");
IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer, downloadParameters.CustomMinimatchOptions);
if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0)
{
items = this.GetFilteredItems(items, minimatcherFuncs);
}
if (!isSingleArtifactDownload && items.Any())
{
Directory.CreateDirectory(rootPath);
}
var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder);
Parallel.ForEach(folderItems, (folder) =>
{
var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
Directory.CreateDirectory(targetPath);
});
var fileItems = items.Where(i => i.ItemType == ContainerItemType.File);
// Only initialize these clients if we know we need to download from Blobstore
// If a client cannot connect to Blobstore, we shouldn't stop them from downloading from FCS
var downloadFromBlob = !AgentKnobs.DisableBuildArtifactsToBlob.GetValue(context).AsBoolean();
DedupStoreClient dedupClient = null;
BlobStoreClientTelemetryTfs clientTelemetry = null;
if (downloadFromBlob && fileItems.Any(x => x.BlobMetadata != null))
{
(dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupClientAsync(
false, (str) => this.tracer.Info(str), this.connection, cancellationToken);
}
var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>(
async item =>
{
var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
var directory = Path.GetDirectoryName(targetPath);
Directory.CreateDirectory(directory);
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
tracer.Info($"Downloading: {targetPath}");
if (item.BlobMetadata != null && downloadFromBlob)
{
await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, dedupClient, clientTelemetry, cancellationToken);
}
else
{
using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken))
using (var targetStream = new FileStream(targetPath, FileMode.Create))
{
await sourceStream.CopyToAsync(targetStream);
}
}
},
maxRetries: downloadParameters.RetryDownloadCount,
cancellationToken: cancellationToken,
tracer: tracer,
continueOnCapturedContext: false,
canRetryDelegate: exception => exception is IOException,
context: null
);
},
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 5000,
MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit,
CancellationToken = cancellationToken,
});
await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken);
// Send results to CustomerIntelligence
if (clientTelemetry != null)
{
var planId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.PlanId)?.Value ?? Guid.Empty.ToString());
var jobId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.JobId)?.Value ?? Guid.Empty.ToString());
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.BuildArtifactDownload,
properties: clientTelemetry.GetArtifactDownloadTelemetry(planId, jobId));
}
// check files (will throw an exception if a file is corrupt)
if (downloadParameters.CheckDownloadedFiles)
{
CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
}
}
public async Task DownloadSingleArtifactAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context)
{
await this.DownloadFileContainerAsync(downloadParameters, buildArtifact, downloadParameters.TargetDirectory, context, cancellationToken);
}
// Download for version 2. This decision was made because version 1 is sealed and we didn't want to break any existing customers.
internal async Task DownloadAsyncV2(
AgentTaskPluginExecutionContext context,
ArtifactDownloadParameters downloadParameters,
DownloadOptions downloadOptions,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
BuildServer buildServer = new BuildServer(connection);
// download all pipeline artifacts if artifact name is missing
if (downloadOptions == DownloadOptions.MultiDownload)
{
List <BuildArtifact> artifacts;
if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId)
{
artifacts = await buildServer.GetArtifactsAsync(downloadParameters.ProjectId, downloadParameters.PipelineId, cancellationToken);
}
else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName)
{
if (string.IsNullOrEmpty(downloadParameters.ProjectName))
{
throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!");
}
else
{
artifacts = await buildServer.GetArtifactsWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, cancellationToken);
}
}
else
{
throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!");
}
IEnumerable <BuildArtifact> buildArtifacts = artifacts.Where(a => string.Equals(a.Resource.Type, PipelineArtifactConstants.Container, StringComparison.OrdinalIgnoreCase));
IEnumerable <BuildArtifact> pipelineArtifacts = artifacts.Where(a => string.Equals(a.Resource.Type, PipelineArtifactConstants.PipelineArtifact, StringComparison.OrdinalIgnoreCase));
IEnumerable <BuildArtifact> fileShareArtifacts = artifacts.Where(a => string.Equals(a.Resource.Type, PipelineArtifactConstants.FileShareArtifact, StringComparison.OrdinalIgnoreCase));
if (buildArtifacts.Any())
{
FileContainerProvider provider = new FileContainerProvider(connection, this.tracer);
await provider.DownloadMultipleArtifactsAsync(downloadParameters, buildArtifacts, cancellationToken, context);
}
if (pipelineArtifacts.Any())
{
PipelineArtifactProvider provider = new PipelineArtifactProvider(context, connection, this.tracer);
await provider.DownloadMultipleArtifactsAsync(downloadParameters, pipelineArtifacts, cancellationToken, context);
}
if (fileShareArtifacts.Any())
{
FileShareProvider provider = new FileShareProvider(context, connection, this.tracer);
await provider.DownloadMultipleArtifactsAsync(downloadParameters, fileShareArtifacts, cancellationToken, context);
}
}
else if (downloadOptions == DownloadOptions.SingleDownload)
{
// 1) get manifest id from artifact data
BuildArtifact buildArtifact;
if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId)
{
buildArtifact = await buildServer.GetArtifact(downloadParameters.ProjectId, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken);
}
else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName)
{
if (string.IsNullOrEmpty(downloadParameters.ProjectName))
{
throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!");
}
else
{
buildArtifact = await buildServer.GetArtifactWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken);
}
}
else
{
throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!");
}
ArtifactProviderFactory factory = new ArtifactProviderFactory(context, connection, this.tracer);
IArtifactProvider provider = factory.GetProvider(buildArtifact);
await provider.DownloadSingleArtifactAsync(downloadParameters, buildArtifact, cancellationToken, context);
}
else
{
throw new InvalidOperationException($"Invalid {nameof(downloadOptions)}!");
}
}
// Download with minimatch patterns, V1.
internal async Task DownloadAsync(
AgentTaskPluginExecutionContext context,
ArtifactDownloadParameters downloadParameters,
DownloadOptions downloadOptions,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
var(dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance
.CreateDedupManifestClientAsync(context.IsSystemDebugTrue(), (str) => context.Output(str), connection, cancellationToken);
BuildServer buildServer = new BuildServer(connection);
using (clientTelemetry)
{
// download all pipeline artifacts if artifact name is missing
if (downloadOptions == DownloadOptions.MultiDownload)
{
List <BuildArtifact> artifacts;
if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId)
{
artifacts = await buildServer.GetArtifactsAsync(downloadParameters.ProjectId, downloadParameters.PipelineId, cancellationToken);
}
else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName)
{
if (string.IsNullOrEmpty(downloadParameters.ProjectName))
{
throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!");
}
else
{
artifacts = await buildServer.GetArtifactsWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, cancellationToken);
}
}
else
{
throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!");
}
IEnumerable <BuildArtifact> pipelineArtifacts = artifacts.Where(a => string.Equals(a.Resource.Type, PipelineArtifactConstants.PipelineArtifact, StringComparison.OrdinalIgnoreCase));
if (pipelineArtifacts.Count() == 0)
{
throw new ArgumentException("Could not find any pipeline artifacts in the build.");
}
else
{
context.Output(StringUtil.Loc("DownloadingMultiplePipelineArtifacts", pipelineArtifacts.Count()));
var artifactNameAndManifestIds = pipelineArtifacts.ToDictionary(
keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong
elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data));
// 2) download to the target path
var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds(
artifactNameAndManifestIds,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters,
minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName,
customMinimatchOptions: downloadParameters.CustomMinimatchOptions);
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true,
context: nameof(DownloadAsync),
cancellationToken: cancellationToken,
continueOnCapturedContext: false);
});
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
}
else if (downloadOptions == DownloadOptions.SingleDownload)
{
// 1) get manifest id from artifact data
BuildArtifact buildArtifact;
if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId)
{
buildArtifact = await buildServer.GetArtifact(downloadParameters.ProjectId, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken);
}
else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName)
{
if (string.IsNullOrEmpty(downloadParameters.ProjectName))
{
throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!");
}
else
{
buildArtifact = await buildServer.GetArtifactWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken);
}
}
else
{
throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!");
}
var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data);
var options = DownloadDedupManifestArtifactOptions.CreateWithManifestId(
manifestId,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters,
customMinimatchOptions: downloadParameters.CustomMinimatchOptions);
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true,
context: nameof(DownloadAsync),
cancellationToken: cancellationToken,
continueOnCapturedContext: false);
});
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
else
{
throw new InvalidOperationException($"Invalid {nameof(downloadOptions)}!");
}
}
}
public async Task DownloadSingleArtifactAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context)
{
await DownloadMultipleArtifactsAsync(downloadParameters, new List <BuildArtifact> {
buildArtifact
}, cancellationToken, context);
}
private async Task DownloadFileContainerAsync(IEnumerable <FileContainerItem> items, ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true)
{
var containerIdAndRoot = ParseContainerId(artifact.Resource.Data);
var projectId = downloadParameters.ProjectId;
tracer.Info($"Start downloading FCS artifact- {artifact.Name}");
if (!isSingleArtifactDownload && items.Any())
{
Directory.CreateDirectory(rootPath);
}
var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder);
Parallel.ForEach(folderItems, (folder) =>
{
var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
Directory.CreateDirectory(targetPath);
});
var fileItems = items.Where(i => i.ItemType == ContainerItemType.File);
// Only initialize these clients if we know we need to download from Blobstore
// If a client cannot connect to Blobstore, we shouldn't stop them from downloading from FCS
var downloadFromBlob = !AgentKnobs.DisableBuildArtifactsToBlob.GetValue(context).AsBoolean();
DedupStoreClient dedupClient = null;
BlobStoreClientTelemetryTfs clientTelemetry = null;
if (downloadFromBlob && fileItems.Any(x => x.BlobMetadata != null))
{
try
{
(dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupClientAsync(
false, (str) => this.tracer.Info(str), this.connection, DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context), cancellationToken);
}
catch (SocketException e)
{
ExceptionsUtil.HandleSocketException(e, connection.Uri.ToString(), context.Warning);
}
catch
{
var blobStoreHost = dedupClient.Client.BaseAddress.Host;
var allowListLink = BlobStoreWarningInfoProvider.GetAllowListLinkForCurrentPlatform();
var warningMessage = StringUtil.Loc("BlobStoreDownloadWarning", blobStoreHost, allowListLink);
// Fall back to streaming through TFS if we cannot reach blobstore
downloadFromBlob = false;
tracer.Warn(warningMessage);
}
}
var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>(
async item =>
{
var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
var directory = Path.GetDirectoryName(targetPath);
Directory.CreateDirectory(directory);
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
tracer.Info($"Downloading: {targetPath}");
if (item.BlobMetadata != null && downloadFromBlob)
{
await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, dedupClient, clientTelemetry, cancellationToken);
}
else
{
using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken))
using (var targetStream = new FileStream(targetPath, FileMode.Create))
{
await sourceStream.CopyToAsync(targetStream);
}
}
},
maxRetries: downloadParameters.RetryDownloadCount,
cancellationToken: cancellationToken,
tracer: tracer,
continueOnCapturedContext: false,
canRetryDelegate: exception => exception is IOException,
context: null
);
},
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 5000,
MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit,
CancellationToken = cancellationToken,
});
await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken);
// Send results to CustomerIntelligence
if (clientTelemetry != null)
{
var planId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.PlanId)?.Value ?? Guid.Empty.ToString());
var jobId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.JobId)?.Value ?? Guid.Empty.ToString());
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.BuildArtifactDownload,
properties: clientTelemetry.GetArtifactDownloadTelemetry(planId, jobId));
}
// check files (will throw an exception if a file is corrupt)
if (downloadParameters.CheckDownloadedFiles)
{
CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
}
}
