// Returns all artifact items. Uses minimatch filters specified in downloadParameters.
private async Task <IEnumerable <FileContainerItem> > GetArtifactItems(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact)
{
(long, string)containerIdAndRoot = ParseContainerId(buildArtifact.Resource.Data);
Guid projectId = downloadParameters.ProjectId;
string[] minimatchPatterns = downloadParameters.MinimatchFilters;
List <FileContainerItem> items = await containerClient.QueryContainerItemsAsync(
containerIdAndRoot.Item1,
projectId,
isShallow : false,
includeBlobMetadata : true,
containerIdAndRoot.Item2
);
IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(
minimatchPatterns,
tracer,
downloadParameters.CustomMinimatchOptions
);
if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0)
{
items = this.GetFilteredItems(items, minimatcherFuncs);
}
return(items);
}
// Returns list of filtered artifact items. Uses minimatch filters specified in downloadParameters.
private async Task <IEnumerable <FileContainerItem> > GetArtifactItems(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact)
{
(long, string)containerIdAndRoot = ParseContainerId(buildArtifact.Resource.Data);
Guid projectId = downloadParameters.ProjectId;
string[] minimatchPatterns = downloadParameters.MinimatchFilters;
List <FileContainerItem> items = await containerClient.QueryContainerItemsAsync(
containerIdAndRoot.Item1,
projectId,
isShallow : false,
includeBlobMetadata : true,
containerIdAndRoot.Item2
);
Options customMinimatchOptions;
if (downloadParameters.CustomMinimatchOptions != null)
{
customMinimatchOptions = downloadParameters.CustomMinimatchOptions;
}
else
{
customMinimatchOptions = new Options()
{
Dot = true,
NoBrace = true,
AllowWindowsPaths = PlatformUtil.RunningOnWindows
};
}
// Getting list of item paths. It is useful to handle list of paths instead of items.
// Also it allows to use the same methods for FileContainerProvider and FileShareProvider.
List <string> paths = new List <string>();
foreach (FileContainerItem item in items)
{
paths.Add(item.Path);
}
ArtifactItemFilters filters = new ArtifactItemFilters(connection, tracer);
Hashtable map = filters.GetMapToFilterItems(paths, minimatchPatterns, customMinimatchOptions);
// Returns filtered list of artifact items. Uses minimatch filters specified in downloadParameters.
List <FileContainerItem> resultItems = filters.ApplyPatternsMapToContainerItems(items, map);
tracer.Info($"{resultItems.Count} final results");
IEnumerable <FileContainerItem> excludedItems = items.Except(resultItems);
foreach (FileContainerItem item in excludedItems)
{
tracer.Info($"Item excluded: {item.Path}");
}
return(resultItems);
}
private async Task DownloadFileContainerAsync(Guid projectId, BuildArtifact artifact, string rootPath, IEnumerable <string> minimatchPatterns, CancellationToken cancellationToken, bool isSingleArtifactDownload = true)
{
var containerIdAndRoot = ParseContainerId(artifact.Resource.Data);
var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, containerIdAndRoot.Item2);
tracer.Info($"Start downloading FCS artifact- {artifact.Name}");
IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer);
if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0)
{
items = this.GetFilteredItems(items, minimatcherFuncs);
}
if (!isSingleArtifactDownload && items.Any())
{
Directory.CreateDirectory(rootPath);
}
var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder);
Parallel.ForEach(folderItems, (folder) =>
{
var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2);
Directory.CreateDirectory(targetPath);
});
var fileItems = items.Where(i => i.ItemType == ContainerItemType.File);
var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>(
async item =>
{
var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2);
var directory = Path.GetDirectoryName(targetPath);
Directory.CreateDirectory(directory);
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken))
{
tracer.Info($"Downloading: {targetPath}");
using (var targetStream = new FileStream(targetPath, FileMode.Create))
{
await sourceStream.CopyToAsync(targetStream);
}
}
},
maxRetries: 5,
cancellationToken: cancellationToken,
tracer: tracer,
continueOnCapturedContext: false,
canRetryDelegate: exception => exception is IOException,
context: null
);
},
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 5000,
MaxDegreeOfParallelism = 8,
CancellationToken = cancellationToken,
});
await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken);
}
private async Task DownloadFileContainerAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true)
{
var containerIdAndRoot = ParseContainerId(artifact.Resource.Data);
var projectId = downloadParameters.ProjectId;
var minimatchPatterns = downloadParameters.MinimatchFilters;
var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, isShallow : false, includeBlobMetadata : true, containerIdAndRoot.Item2);
tracer.Info($"Start downloading FCS artifact- {artifact.Name}");
IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer);
if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0)
{
items = this.GetFilteredItems(items, minimatcherFuncs);
}
if (!isSingleArtifactDownload && items.Any())
{
Directory.CreateDirectory(rootPath);
}
var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder);
Parallel.ForEach(folderItems, (folder) =>
{
var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
Directory.CreateDirectory(targetPath);
});
var fileItems = items.Where(i => i.ItemType == ContainerItemType.File);
var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>(
async item =>
{
var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
var directory = Path.GetDirectoryName(targetPath);
Directory.CreateDirectory(directory);
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
tracer.Info($"Downloading: {targetPath}");
if (item.BlobMetadata != null)
{
await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, cancellationToken);
}
else
{
using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken))
using (var targetStream = new FileStream(targetPath, FileMode.Create))
{
await sourceStream.CopyToAsync(targetStream);
}
}
},
maxRetries: downloadParameters.RetryDownloadCount,
cancellationToken: cancellationToken,
tracer: tracer,
continueOnCapturedContext: false,
canRetryDelegate: exception => exception is IOException,
context: null
);
},
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 5000,
MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit,
CancellationToken = cancellationToken,
});
await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken);
// check files (will throw an exception if a file is corrupt)
if (downloadParameters.CheckDownloadedFiles)
{
CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
}
}
private async Task DownloadFileContainerAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true)
{
var containerIdAndRoot = ParseContainerId(artifact.Resource.Data);
var projectId = downloadParameters.ProjectId;
var minimatchPatterns = downloadParameters.MinimatchFilters;
var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, isShallow : false, includeBlobMetadata : true, containerIdAndRoot.Item2);
tracer.Info($"Start downloading FCS artifact- {artifact.Name}");
IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer, downloadParameters.CustomMinimatchOptions);
if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0)
{
items = this.GetFilteredItems(items, minimatcherFuncs);
}
if (!isSingleArtifactDownload && items.Any())
{
Directory.CreateDirectory(rootPath);
}
var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder);
Parallel.ForEach(folderItems, (folder) =>
{
var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
Directory.CreateDirectory(targetPath);
});
var fileItems = items.Where(i => i.ItemType == ContainerItemType.File);
// Only initialize these clients if we know we need to download from Blobstore
// If a client cannot connect to Blobstore, we shouldn't stop them from downloading from FCS
var downloadFromBlob = !AgentKnobs.DisableBuildArtifactsToBlob.GetValue(context).AsBoolean();
DedupStoreClient dedupClient = null;
BlobStoreClientTelemetryTfs clientTelemetry = null;
if (downloadFromBlob && fileItems.Any(x => x.BlobMetadata != null))
{
(dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupClientAsync(
false, (str) => this.tracer.Info(str), this.connection, cancellationToken);
}
var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>(
async item =>
{
var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
var directory = Path.GetDirectoryName(targetPath);
Directory.CreateDirectory(directory);
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
tracer.Info($"Downloading: {targetPath}");
if (item.BlobMetadata != null && downloadFromBlob)
{
await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, dedupClient, clientTelemetry, cancellationToken);
}
else
{
using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken))
using (var targetStream = new FileStream(targetPath, FileMode.Create))
{
await sourceStream.CopyToAsync(targetStream);
}
}
},
maxRetries: downloadParameters.RetryDownloadCount,
cancellationToken: cancellationToken,
tracer: tracer,
continueOnCapturedContext: false,
canRetryDelegate: exception => exception is IOException,
context: null
);
},
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 5000,
MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit,
CancellationToken = cancellationToken,
});
await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken);
// Send results to CustomerIntelligence
if (clientTelemetry != null)
{
var planId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.PlanId)?.Value ?? Guid.Empty.ToString());
var jobId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.JobId)?.Value ?? Guid.Empty.ToString());
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.BuildArtifactDownload,
properties: clientTelemetry.GetArtifactDownloadTelemetry(planId, jobId));
}
// check files (will throw an exception if a file is corrupt)
if (downloadParameters.CheckDownloadedFiles)
{
CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
}
}
public async Task DownloadFromContainerAsync(
RunnerActionPluginExecutionContext context,
String destination,
CancellationToken cancellationToken)
{
// Find out all container items need to be processed
List <FileContainerItem> containerItems = new List <FileContainerItem>();
int retryCount = 0;
while (retryCount < 3)
{
try
{
containerItems = await _fileContainerHttpClient.QueryContainerItemsAsync(_containerId,
_projectId,
_containerPath,
cancellationToken : cancellationToken);
break;
}
catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
{
context.Debug($"Container query has been cancelled.");
throw;
}
catch (Exception ex) when(retryCount < 2)
{
retryCount++;
context.Warning($"Fail to query container items under #/{_containerId}/{_containerPath}, Error: {ex.Message}");
context.Debug(ex.ToString());
}
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
context.Warning($"Back off {backOff.TotalSeconds} seconds before retry.");
await Task.Delay(backOff);
}
if (containerItems.Count == 0)
{
context.Output($"There is nothing under #/{_containerId}/{_containerPath}");
return;
}
// container items will include both folders, files and even file with zero size
// Create all required empty folders and emptry files, gather a list of files that we need to download from server.
int foldersCreated = 0;
int emptryFilesCreated = 0;
List <DownloadInfo> downloadFiles = new List <DownloadInfo>();
foreach (var item in containerItems.OrderBy(x => x.Path))
{
if (!item.Path.StartsWith(_containerPath, StringComparison.OrdinalIgnoreCase))
{
throw new ArgumentOutOfRangeException($"Item {item.Path} is not under #/{_containerId}/{_containerPath}");
}
var localRelativePath = item.Path.Substring(_containerPath.Length).TrimStart('/');
var localPath = Path.Combine(destination, localRelativePath);
if (item.ItemType == ContainerItemType.Folder)
{
context.Debug($"Ensure folder exists: {localPath}");
Directory.CreateDirectory(localPath);
foldersCreated++;
}
else if (item.ItemType == ContainerItemType.File)
{
if (item.FileLength == 0)
{
context.Debug($"Create empty file at: {localPath}");
var parentDirectory = Path.GetDirectoryName(localPath);
Directory.CreateDirectory(parentDirectory);
IOUtil.DeleteFile(localPath);
using (new FileStream(localPath, FileMode.Create))
{
}
emptryFilesCreated++;
}
else
{
context.Debug($"Prepare download {item.Path} to {localPath}");
downloadFiles.Add(new DownloadInfo(item.Path, localPath));
}
}
else
{
throw new NotSupportedException(item.ItemType.ToString());
}
}
if (foldersCreated > 0)
{
context.Output($"{foldersCreated} folders created.");
}
if (emptryFilesCreated > 0)
{
context.Output($"{emptryFilesCreated} empty files created.");
}
if (downloadFiles.Count == 0)
{
context.Output($"There is nothing to download");
return;
}
// Start multi-task to download all files.
using (_downloadCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
// try download all files for the first time.
DownloadResult downloadResult = await ParallelDownloadAsync(context, downloadFiles.AsReadOnly(), Math.Min(downloadFiles.Count, Environment.ProcessorCount), _downloadCancellationTokenSource.Token);
if (downloadResult.FailedFiles.Count == 0)
{
// all files have been download succeed.
context.Output($"{downloadFiles.Count} files download succeed.");
return;
}
else
{
context.Output($"{downloadResult.FailedFiles.Count} files failed to download, retry these files after a minute.");
}
// Delay 1 min then retry failed files.
for (int timer = 60; timer > 0; timer -= 5)
{
context.Output($"Retry file download after {timer} seconds.");
await Task.Delay(TimeSpan.FromSeconds(5), _uploadCancellationTokenSource.Token);
}
// Retry download all failed files.
context.Output($"Start retry {downloadResult.FailedFiles.Count} failed files upload.");
DownloadResult retryDownloadResult = await ParallelDownloadAsync(context, downloadResult.FailedFiles.AsReadOnly(), Math.Min(downloadResult.FailedFiles.Count, Environment.ProcessorCount), _downloadCancellationTokenSource.Token);
if (retryDownloadResult.FailedFiles.Count == 0)
{
// all files have been download succeed after retry.
context.Output($"{downloadResult.FailedFiles} files download succeed after retry.");
return;
}
else
{
throw new Exception($"{retryDownloadResult.FailedFiles.Count} files failed to download even after retry.");
}
}
}