private async Task DownloadFileShareArtifactAsync(string sourcePath, string destPath, int parallelCount, CancellationToken cancellationToken, IEnumerable <Func <string, bool> > minimatchFuncs = null)
{
var trimChars = new[] { '\\', '/' };
sourcePath = sourcePath.TrimEnd(trimChars);
IEnumerable <FileInfo> files =
new DirectoryInfo(sourcePath).EnumerateFiles("*", SearchOption.AllDirectories);
var parallelism = new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = parallelCount,
BoundedCapacity = 2 * parallelCount,
CancellationToken = cancellationToken
};
var actionBlock = NonSwallowingActionBlock.Create <FileInfo>(
action: async file =>
{
if (minimatchFuncs == null || minimatchFuncs.Any(match => match(file.FullName)))
{
string tempPath = Path.Combine(destPath, Path.GetRelativePath(sourcePath, file.FullName));
context.Output(StringUtil.Loc("CopyFileToDestination", file, tempPath));
FileInfo tempFile = new System.IO.FileInfo(tempPath);
using (StreamReader fileReader = GetFileReader(file.FullName))
{
await WriteStreamToFile(
fileReader.BaseStream,
tempFile.FullName,
DefaultStreamBufferSize,
cancellationToken);
}
}
},
dataflowBlockOptions: parallelism);
await actionBlock.SendAllAndCompleteAsync(files, actionBlock, cancellationToken);
}
private static async Task <List <BlobFileInfo> > GenerateHashes(IReadOnlyList <string> filePaths, CancellationToken cancellationToken)
{
var nodes = new BlobFileInfo[filePaths.Count];
var queue = NonSwallowingActionBlock.Create <int>(
async i =>
{
var itemPath = filePaths[i];
try
{
var dedupNode = await ChunkerHelper.CreateFromFileAsync(FileSystem.Instance, itemPath, cancellationToken, false);
nodes[i] = new BlobFileInfo
{
Path = itemPath,
Node = dedupNode,
Success = dedupNode != null
};
}
catch (Exception)
{
nodes[i] = new BlobFileInfo
{
Path = itemPath,
Success = false
};
}
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount,
CancellationToken = cancellationToken,
});
await queue.SendAllAndCompleteSingleBlockNetworkAsync(Enumerable.Range(0, filePaths.Count), cancellationToken);
return(nodes.ToList());
}
private async Task <ArtifactRecord> DownloadFileShareArtifactAsync(
string sourcePath,
string destPath,
int parallelCount,
CancellationToken cancellationToken,
IEnumerable <string> minimatchPatterns = null)
{
Stopwatch watch = Stopwatch.StartNew();
IEnumerable <Func <string, bool> > minimatchFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, this.tracer);
var trimChars = new[] { '\\', '/' };
sourcePath = sourcePath.TrimEnd(trimChars);
var artifactName = new DirectoryInfo(destPath).Name;
IEnumerable <FileInfo> files =
new DirectoryInfo(sourcePath).EnumerateFiles("*", SearchOption.AllDirectories);
var parallelism = new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = parallelCount,
BoundedCapacity = 2 * parallelCount,
CancellationToken = cancellationToken
};
var contentSize = 0;
var fileCount = 0;
var actionBlock = NonSwallowingActionBlock.Create <FileInfo>(
action: async file =>
{
if (minimatchFuncs == null || minimatchFuncs.Any(match => match(file.FullName)))
{
string tempPath = Path.Combine(destPath, Path.GetRelativePath(sourcePath, file.FullName));
context.Output(StringUtil.Loc("CopyFileToDestination", file, tempPath));
FileInfo tempFile = new System.IO.FileInfo(tempPath);
using (StreamReader fileReader = GetFileReader(file.FullName))
{
await WriteStreamToFile(
fileReader.BaseStream,
tempFile.FullName,
DefaultStreamBufferSize,
cancellationToken);
}
Interlocked.Add(ref contentSize, tempPath.Length);
Interlocked.Increment(ref fileCount);
}
},
dataflowBlockOptions: parallelism);
await actionBlock.SendAllAndCompleteAsync(files, actionBlock, cancellationToken);
watch.Stop();
return(new ArtifactRecord(artifactName,
fileCount,
contentSize,
watch.ElapsedMilliseconds));
}
private async Task DownloadFileContainerAsync(Guid projectId, BuildArtifact artifact, string rootPath, IEnumerable <string> minimatchPatterns, CancellationToken cancellationToken, bool isSingleArtifactDownload = true)
{
var containerIdAndRoot = ParseContainerId(artifact.Resource.Data);
var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, containerIdAndRoot.Item2);
tracer.Info($"Start downloading FCS artifact- {artifact.Name}");
IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer);
if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0)
{
items = this.GetFilteredItems(items, minimatcherFuncs);
}
if (!isSingleArtifactDownload && items.Any())
{
Directory.CreateDirectory(rootPath);
}
var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder);
Parallel.ForEach(folderItems, (folder) =>
{
var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2);
Directory.CreateDirectory(targetPath);
});
var fileItems = items.Where(i => i.ItemType == ContainerItemType.File);
var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>(
async item =>
{
var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2);
var directory = Path.GetDirectoryName(targetPath);
Directory.CreateDirectory(directory);
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken))
{
tracer.Info($"Downloading: {targetPath}");
using (var targetStream = new FileStream(targetPath, FileMode.Create))
{
await sourceStream.CopyToAsync(targetStream);
}
}
},
maxRetries: 5,
cancellationToken: cancellationToken,
tracer: tracer,
continueOnCapturedContext: false,
canRetryDelegate: exception => exception is IOException,
context: null
);
},
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 5000,
MaxDegreeOfParallelism = 8,
CancellationToken = cancellationToken,
});
await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken);
}
private async Task DownloadFileContainerAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true)
{
var containerIdAndRoot = ParseContainerId(artifact.Resource.Data);
var projectId = downloadParameters.ProjectId;
var minimatchPatterns = downloadParameters.MinimatchFilters;
var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, isShallow : false, includeBlobMetadata : true, containerIdAndRoot.Item2);
tracer.Info($"Start downloading FCS artifact- {artifact.Name}");
IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer);
if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0)
{
items = this.GetFilteredItems(items, minimatcherFuncs);
}
if (!isSingleArtifactDownload && items.Any())
{
Directory.CreateDirectory(rootPath);
}
var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder);
Parallel.ForEach(folderItems, (folder) =>
{
var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
Directory.CreateDirectory(targetPath);
});
var fileItems = items.Where(i => i.ItemType == ContainerItemType.File);
var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>(
async item =>
{
var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
var directory = Path.GetDirectoryName(targetPath);
Directory.CreateDirectory(directory);
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
tracer.Info($"Downloading: {targetPath}");
if (item.BlobMetadata != null)
{
await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, cancellationToken);
}
else
{
using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken))
using (var targetStream = new FileStream(targetPath, FileMode.Create))
{
await sourceStream.CopyToAsync(targetStream);
}
}
},
maxRetries: downloadParameters.RetryDownloadCount,
cancellationToken: cancellationToken,
tracer: tracer,
continueOnCapturedContext: false,
canRetryDelegate: exception => exception is IOException,
context: null
);
},
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 5000,
MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit,
CancellationToken = cancellationToken,
});
await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken);
// check files (will throw an exception if a file is corrupt)
if (downloadParameters.CheckDownloadedFiles)
{
CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
}
}
private async Task DownloadFileContainerAsync(IEnumerable <FileContainerItem> items, ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true)
{
var containerIdAndRoot = ParseContainerId(artifact.Resource.Data);
var projectId = downloadParameters.ProjectId;
tracer.Info($"Start downloading FCS artifact- {artifact.Name}");
if (!isSingleArtifactDownload && items.Any())
{
Directory.CreateDirectory(rootPath);
}
var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder);
Parallel.ForEach(folderItems, (folder) =>
{
var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
Directory.CreateDirectory(targetPath);
});
var fileItems = items.Where(i => i.ItemType == ContainerItemType.File);
// Only initialize these clients if we know we need to download from Blobstore
// If a client cannot connect to Blobstore, we shouldn't stop them from downloading from FCS
var downloadFromBlob = !AgentKnobs.DisableBuildArtifactsToBlob.GetValue(context).AsBoolean();
DedupStoreClient dedupClient = null;
BlobStoreClientTelemetryTfs clientTelemetry = null;
if (downloadFromBlob && fileItems.Any(x => x.BlobMetadata != null))
{
try
{
(dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupClientAsync(
false, (str) => this.tracer.Info(str), this.connection, DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context), cancellationToken);
}
catch (SocketException e)
{
ExceptionsUtil.HandleSocketException(e, connection.Uri.ToString(), context.Warning);
}
catch
{
var blobStoreHost = dedupClient.Client.BaseAddress.Host;
var allowListLink = BlobStoreWarningInfoProvider.GetAllowListLinkForCurrentPlatform();
var warningMessage = StringUtil.Loc("BlobStoreDownloadWarning", blobStoreHost, allowListLink);
// Fall back to streaming through TFS if we cannot reach blobstore
downloadFromBlob = false;
tracer.Warn(warningMessage);
}
}
var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>(
async item =>
{
var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
var directory = Path.GetDirectoryName(targetPath);
Directory.CreateDirectory(directory);
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
tracer.Info($"Downloading: {targetPath}");
if (item.BlobMetadata != null && downloadFromBlob)
{
await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, dedupClient, clientTelemetry, cancellationToken);
}
else
{
using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken))
using (var targetStream = new FileStream(targetPath, FileMode.Create))
{
await sourceStream.CopyToAsync(targetStream);
}
}
},
maxRetries: downloadParameters.RetryDownloadCount,
cancellationToken: cancellationToken,
tracer: tracer,
continueOnCapturedContext: false,
canRetryDelegate: exception => exception is IOException,
context: null
);
},
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 5000,
MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit,
CancellationToken = cancellationToken,
});
await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken);
// Send results to CustomerIntelligence
if (clientTelemetry != null)
{
var planId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.PlanId)?.Value ?? Guid.Empty.ToString());
var jobId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.JobId)?.Value ?? Guid.Empty.ToString());
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.BuildArtifactDownload,
properties: clientTelemetry.GetArtifactDownloadTelemetry(planId, jobId));
}
// check files (will throw an exception if a file is corrupt)
if (downloadParameters.CheckDownloadedFiles)
{
CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
}
}
private async Task DownloadFileContainerAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true)
{
var containerIdAndRoot = ParseContainerId(artifact.Resource.Data);
var projectId = downloadParameters.ProjectId;
var minimatchPatterns = downloadParameters.MinimatchFilters;
var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, isShallow : false, includeBlobMetadata : true, containerIdAndRoot.Item2);
tracer.Info($"Start downloading FCS artifact- {artifact.Name}");
IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer, downloadParameters.CustomMinimatchOptions);
if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0)
{
items = this.GetFilteredItems(items, minimatcherFuncs);
}
if (!isSingleArtifactDownload && items.Any())
{
Directory.CreateDirectory(rootPath);
}
var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder);
Parallel.ForEach(folderItems, (folder) =>
{
var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
Directory.CreateDirectory(targetPath);
});
var fileItems = items.Where(i => i.ItemType == ContainerItemType.File);
// Only initialize these clients if we know we need to download from Blobstore
// If a client cannot connect to Blobstore, we shouldn't stop them from downloading from FCS
var downloadFromBlob = !AgentKnobs.DisableBuildArtifactsToBlob.GetValue(context).AsBoolean();
DedupStoreClient dedupClient = null;
BlobStoreClientTelemetryTfs clientTelemetry = null;
if (downloadFromBlob && fileItems.Any(x => x.BlobMetadata != null))
{
(dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupClientAsync(
false, (str) => this.tracer.Info(str), this.connection, cancellationToken);
}
var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>(
async item =>
{
var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
var directory = Path.GetDirectoryName(targetPath);
Directory.CreateDirectory(directory);
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
tracer.Info($"Downloading: {targetPath}");
if (item.BlobMetadata != null && downloadFromBlob)
{
await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, dedupClient, clientTelemetry, cancellationToken);
}
else
{
using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken))
using (var targetStream = new FileStream(targetPath, FileMode.Create))
{
await sourceStream.CopyToAsync(targetStream);
}
}
},
maxRetries: downloadParameters.RetryDownloadCount,
cancellationToken: cancellationToken,
tracer: tracer,
continueOnCapturedContext: false,
canRetryDelegate: exception => exception is IOException,
context: null
);
},
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 5000,
MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit,
CancellationToken = cancellationToken,
});
await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken);
// Send results to CustomerIntelligence
if (clientTelemetry != null)
{
var planId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.PlanId)?.Value ?? Guid.Empty.ToString());
var jobId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.JobId)?.Value ?? Guid.Empty.ToString());
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.BuildArtifactDownload,
properties: clientTelemetry.GetArtifactDownloadTelemetry(planId, jobId));
}
// check files (will throw an exception if a file is corrupt)
if (downloadParameters.CheckDownloadedFiles)
{
CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
}
}
private async Task <ArtifactRecord> DownloadFileShareArtifactAsync(
string sourcePath,
string destPath,
int parallelCount,
ArtifactDownloadParameters downloadParameters,
CancellationToken cancellationToken,
IEnumerable <string> minimatchPatterns = null)
{
Stopwatch watch = Stopwatch.StartNew();
IEnumerable <Func <string, bool> > minimatchFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, this.tracer);
var trimChars = new[] { '\\', '/' };
sourcePath = sourcePath.TrimEnd(trimChars);
var artifactName = new DirectoryInfo(sourcePath).Name;
List <FileInfo> files =
new DirectoryInfo(sourcePath).EnumerateFiles("*", SearchOption.AllDirectories).ToList <FileInfo>();
ArtifactItemFilters filters = new ArtifactItemFilters(connection, tracer);
// Getting list of file paths. It is useful to handle list of paths instead of files.
// Also it allows to use the same methods for FileContainerProvider and FileShareProvider.
List <string> paths = new List <string>();
foreach (FileInfo file in files)
{
string pathInArtifact = filters.RemoveSourceDirFromPath(file, sourcePath);
paths.Add(Path.Combine(artifactName, pathInArtifact));
}
Options customMinimatchOptions;
if (downloadParameters.CustomMinimatchOptions != null)
{
customMinimatchOptions = downloadParameters.CustomMinimatchOptions;
}
else
{
customMinimatchOptions = new Options()
{
Dot = true,
NoBrace = true,
AllowWindowsPaths = PlatformUtil.RunningOnWindows
};
}
Hashtable map = filters.GetMapToFilterItems(paths, downloadParameters.MinimatchFilters, customMinimatchOptions);
// Returns filtered list of artifact items. Uses minimatch filters specified in downloadParameters.
IEnumerable <FileInfo> filteredFiles = filters.ApplyPatternsMapToFileShareItems(files, map, sourcePath);
tracer.Info($"{filteredFiles.ToList().Count} final results");
IEnumerable <FileInfo> excludedItems = files.Except(filteredFiles);
foreach (FileInfo item in excludedItems)
{
tracer.Info($"File excluded: {item.FullName}");
}
var parallelism = new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = parallelCount,
BoundedCapacity = 2 * parallelCount,
CancellationToken = cancellationToken
};
var contentSize = 0;
var fileCount = 0;
var actionBlock = NonSwallowingActionBlock.Create <FileInfo>(
action: async file =>
{
if (minimatchFuncs == null || minimatchFuncs.Any(match => match(file.FullName)))
{
string tempPath = Path.Combine(destPath, Path.GetRelativePath(sourcePath, file.FullName));
context.Output(StringUtil.Loc("CopyFileToDestination", file, tempPath));
FileInfo tempFile = new System.IO.FileInfo(tempPath);
using (StreamReader fileReader = GetFileReader(file.FullName))
{
await WriteStreamToFile(
fileReader.BaseStream,
tempFile.FullName,
DefaultStreamBufferSize,
cancellationToken);
}
Interlocked.Add(ref contentSize, tempPath.Length);
Interlocked.Increment(ref fileCount);
}
},
dataflowBlockOptions: parallelism);
await actionBlock.SendAllAndCompleteAsync(filteredFiles, actionBlock, cancellationToken);
watch.Stop();
return(new ArtifactRecord(artifactName,
fileCount,
contentSize,
watch.ElapsedMilliseconds));
}
