// Download with minimatch patterns.
internal async Task DownloadAsyncMinimatch(
AgentTaskPluginExecutionContext context,
Guid projectId,
int buildId,
string artifactName,
string targetDir,
string[] minimatchFilters,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
// 1) get manifest id from artifact data
BuildServer buildHelper = new BuildServer(connection);
BuildArtifact art = await buildHelper.GetArtifact(projectId, buildId, artifactName, cancellationToken);
if (art.Resource.Type != "PipelineArtifact")
{
throw new ArgumentException($"The artifact is not of the type Pipeline Artifact. Unrecognized type: {art.Resource.Type}.");
}
var manifestId = DedupIdentifier.Create(art.Resource.Data);
// 2) download to the target path
var buildDropManager = this.GetBDM(context, connection);
DownloadPipelineArtifactOptions options = DownloadPipelineArtifactOptions.CreateWithManifestId(manifestId, targetDir, proxyUri: null, minimatchPatterns: minimatchFilters);
await buildDropManager.DownloadAsync(options, cancellationToken);
}
/// <summary>
/// Create an artifact link, such as a file or folder path or a version control path.
/// </summary>
/// <param name="name">The artifact name.</param>
/// <param name="type">The artifact type.
/// The following artifact types are supported:
/// <list type="table">
/// <listheader>
/// <term>Type</term>
/// <term>Description</term>
/// </listheader>
/// <item>
/// <term>FilePath</term>
/// <term>File path type</term>
/// </item>
/// <item>
/// <term>GitRef</term>
/// <term>Git reference type</term>
/// </item>
/// <item>
/// <term>TFVCLabel</term>
/// <term>TFVC label type</term>
/// </item>
/// <item>
/// <term>VersionControl</term>
/// <term>Version control path type</term>
/// </item>
/// </list>
/// </param>
/// <param name="location">The link path or value.</param>
/// <returns>Description of the new artifact or <c>null</c> if no build could be found and
/// <see cref="AzureDevOpsBuildSettings.ThrowExceptionIfBuildCouldNotBeFound"/> is set to <c>false</c>.</returns>
/// <exception cref="AzureDevOpsBuildNotFoundException">If build could not be found and
/// <see cref="AzureDevOpsBuildSettings.ThrowExceptionIfBuildCouldNotBeFound"/> is set to <c>true</c>.</exception>
public AzureDevOpsBuildArtifact LinkArtifact(string name, string type, string location)
{
if (!this.ValidateBuild())
{
return(null);
}
using (var buildClient = this.buildClientFactory.CreateBuildClient(this.CollectionUrl, this.credentials))
{
var artifact =
new BuildArtifact
{
Name = name,
Resource =
new ArtifactResource
{
Type = type,
Data = location,
},
};
return
(buildClient
.CreateArtifactAsync(artifact, this.ProjectId, this.BuildId)
.ConfigureAwait(false)
.GetAwaiter()
.GetResult()
.ToAzureDevOpsBuildArtifact());
}
}
// Download pipeline artifact from VSTS BlobStore service through BuildDropManager to a target path
internal async Task DownloadAsync(
AgentTaskPluginExecutionContext context,
Guid projectId,
int buildId,
string artifactName,
string targetDir,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
// 1) get manifest id from artifact data
BuildServer buildHelper = new BuildServer(connection);
BuildArtifact art = await buildHelper.GetArtifact(projectId, buildId, artifactName, cancellationToken);
var manifestId = DedupIdentifier.Create(art.Resource.Data);
// 2) download to the target path
var httpclient = connection.GetClient <DedupStoreHttpClient>();
var tracer = new CallbackAppTraceSource(str => context.Output(str), System.Diagnostics.SourceLevels.Information);
httpclient.SetTracer(tracer);
var client = new DedupStoreClientWithDataport(httpclient, maxParallelism: 16 * Environment.ProcessorCount);
var BuildDropManager = new BuildDropManager(client, tracer);
await BuildDropManager.DownloadAsync(manifestId, targetDir, cancellationToken);
}
private static BuildArtifact getArtifact(string resultKey, XPathNavigator artifact)
{
if (!artifact.HasChildren)
{
return(null);
}
XPathNavigator art = artifact.Clone();
art.MoveToFirstChild();
string name = null;
string link = null;
do
{
switch (art.Name)
{
case "name":
name = art.Value;
break;
case "link":
link = XPathUtils.getAttributeSafely(art, "href", null);
break;
}
} while (art.MoveToNext());
if (name == null | link == null)
{
return(null);
}
BuildArtifact a = new BuildArtifact(resultKey, name, link);
return(a);
}
// Upload from target path to Azure DevOps BlobStore service through DedupManifestArtifactClient, then associate it with the build
internal async Task UploadAsync(
AgentTaskPluginExecutionContext context,
Guid projectId,
int pipelineId,
string name,
string source,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry);
using (clientTelemetry)
{
//Upload the pipeline artifact.
PipelineArtifactActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(UploadAsync), context));
PublishResult result = await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() =>
{
return(await dedupManifestClient.PublishAsync(source, cancellationToken));
}
);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: uploadRecord);
// 2) associate the pipeline artifact with an build artifact
BuildServer buildHelper = new BuildServer(connection);
Dictionary <string, string> propertiesDictionary = new Dictionary <string, string>();
propertiesDictionary.Add(PipelineArtifactConstants.RootId, result.RootId.ValueString);
propertiesDictionary.Add(PipelineArtifactConstants.ProofNodes, StringUtil.ConvertToJson(result.ProofNodes.ToArray()));
propertiesDictionary.Add(PipelineArtifactConstants.ArtifactSize, result.ContentSize.ToString());
BuildArtifact buildArtifact = await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
return(await buildHelper.AssociateArtifactAsync(projectId,
pipelineId,
name,
context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.JobId)?.Value ?? string.Empty,
ArtifactResourceTypes.PipelineArtifact,
result.ManifestId.ValueString,
propertiesDictionary,
cancellationToken));
},
maxRetries : 3,
tracer : tracer,
canRetryDelegate : e => e is TimeoutException || e.InnerException is TimeoutException,
cancellationToken : cancellationToken,
continueOnCapturedContext : false);
context.Output(StringUtil.Loc("AssociateArtifactWithBuild", buildArtifact.Id, pipelineId));
}
}
/// <summary>
/// Converts a <see cref="BuildArtifact"/> to an <see cref="AzureDevOpsBuildArtifact"/>.
/// </summary>
/// <param name="buildArtifact">Build artifact record to convert.</param>
/// <returns>Converted build artifact record.</returns>
public static AzureDevOpsBuildArtifact ToAzureDevOpsBuildArtifact(this BuildArtifact buildArtifact)
{
buildArtifact.NotNull(nameof(buildArtifact));
return
(new AzureDevOpsBuildArtifact
{
Id = buildArtifact.Id,
Name = buildArtifact.Name,
Resource = buildArtifact.Resource.ToAzureDevOpsArtifactResource(),
});
}
private async Task ProcessArtifactAsync(BuildSyncContext context, BuildArtifact artifact, CancellationToken stoppingToken)
{
// Stream the artifact down to disk
var artifactFile = await SaveArtifactAsync(artifact, stoppingToken);
try
{
using var archive = ZipFile.OpenRead(artifactFile);
foreach (var file in archive.Entries)
{
using (_logger.BeginScope("File: {ArtifactFile}", file.FullName))
{
if (file.Name.EndsWith(".xml"))
{
_logger.LogTrace(new EventId(0, "ProcessingFile"), "Processing file {ArtifactFile}...", file);
await ProcessTestResultFileAsync(context, file, stoppingToken);
_logger.LogTrace(new EventId(0, "ProcessedFile"), "Processed file {ArtifactFile}.", file);
}
else
{
_logger.LogTrace(new EventId(0, "SkippingFile"), "Skipping artifact file {ArtifactFile}.", file);
}
}
}
}
catch (OperationCanceledException)
{
_logger.LogDebug(new EventId(0, "CancellingArtifactProcessing"), "Cancelling processing of artifact: {ArtifactName}.", artifact.Name);
throw;
}
catch (Exception ex)
{
_logger.LogError(new EventId(0, "ErrorProcessingArtifact"), ex, "Error processing artifact: {ArtifactName}.", artifact.Name);
// We don't want to continue processing this build if an artifact failed. That way we can try again later.
throw;
}
finally
{
try
{
_logger.LogTrace(new EventId(0, "DeletingTemporaryFile"), "Deleting temporary file {TempFile} for Artifact {ArtifactName}.", artifactFile, artifact.Name);
File.Delete(artifactFile);
}
catch (Exception ex)
{
_logger.LogWarning(new EventId(0, "ErrorDeletingTemporaryFile"), ex, "Error deleting temporary file: {TempFile}.", artifactFile);
}
}
}
/// <summary>
/// Download a build content
/// </summary>
/// <param name="StartedBuild"></param>
/// <param name="ArtifactName"></param>
private static void DownloadBuildResults(Build StartedBuild, string ArtifactName)
{
BuildArtifact drop = BuildClient.GetArtifactAsync(StartedBuild.Id, ArtifactName).Result; //get detiled info
Console.WriteLine("Build location: " + drop.Resource.DownloadUrl);
string dropFileName = String.Format("{0}_{1}.zip", StartedBuild.Definition.Name, StartedBuild.BuildNumber);
Stream zipStream = BuildClient.GetArtifactContentZipAsync(StartedBuild.Id, ArtifactName).Result; //get content
using (FileStream zipFile = new FileStream(dropFileName, FileMode.Create))
zipStream.CopyTo(zipFile);
Console.WriteLine("The file '{0}' has been downloaded.", dropFileName);
}
public static BuildArtifactKind GetArtifactKind(BuildArtifact buildArtifact)
{
if (buildArtifact.Resource is object &&
buildArtifact.Resource.Type is object)
{
try
{
return((BuildArtifactKind)Enum.Parse(typeof(BuildArtifactKind), buildArtifact.Resource.Type));
}
catch
{
return(BuildArtifactKind.Unknown);
}
}
return(BuildArtifactKind.Unknown);
}
public Default()
{
Defaults.FrameworkVersion = FrameworkVersion.NET3_5;
dirBase = new BuildFolder(Environment.CurrentDirectory);
dirCompile = dirBase.SubFolder("compile");
dirTools = dirBase.SubFolder("tools");
dirInstall = new BuildFolder(@"C:\Program Files (x86)\CruiseControl.NET\server");
assemblyFluentBuild = dirTools.SubFolder("FluentBuild").File("fluentbuild.dll");
assemblyThoughtworksCore = dirTools.SubFolder("CruiseControl").File("ThoughtWorks.CruiseControl.Core.dll");
assemblyNetReflector = dirTools.SubFolder("CruiseControl").File("NetReflector.dll");
assemblyCompiledPlugin = dirCompile.File("ccnet.fluentbuild.plugin.dll");
AddTask(Clean);
AddTask(Uninstall);
AddTask(Compile);
AddTask(Install);
}
/// <summary>
/// Returns the uncompressed byte size of the artifact
/// </summary>
public static int?GetArtifactByteSize(BuildArtifact buildArtifact)
{
if (buildArtifact.Resource is object &&
buildArtifact.Resource.Properties is object)
{
try
{
dynamic properties = buildArtifact.Resource.Properties;
return((int)properties.artifactsize);
}
catch
{
// Okay if dynamic information is not available
return(null);
}
}
return(null);
}
public IArtifactProvider GetProvider(BuildArtifact buildArtifact)
{
IArtifactProvider provider;
string artifactType = buildArtifact.Resource.Type;
switch (artifactType)
{
case PipelineArtifactConstants.PipelineArtifact:
provider = pipelineArtifactProvider;
break;
case PipelineArtifactConstants.Container:
provider = fileContainerProvider;
break;
default:
throw new InvalidOperationException($"{buildArtifact} is neither of type PipelineArtifact nor BuildArtifact");
}
return(provider);
}
public async Task TestDownloadArtifactAsync()
{
byte[] sourceContent1 = GenerateRandomData();
byte[] sourceContent2 = GenerateRandomData();
TestFile sourceFile1 = new TestFile(sourceContent1);
TestFile sourceFile2 = new TestFile(sourceContent2);
sourceFile1.PlaceItem(Path.Combine(Directory.GetCurrentDirectory(), Path.Combine(TestDownloadSourceFolder, "drop/test2.txt")));
sourceFile2.PlaceItem(Path.Combine(Directory.GetCurrentDirectory(), Path.Combine(TestDownloadSourceFolder, "drop/test3.txt")));
using (var hostContext = new TestHostContext(this))
{
var context = new AgentTaskPluginExecutionContext(hostContext.GetTrace());
var provider = new FileShareProvider(context, null, context.CreateArtifactsTracer(), new MockDedupManifestArtifactClientFactory());
string sourcePath = Path.Combine(Directory.GetCurrentDirectory(), TestDownloadSourceFolder);
string destPath = Path.Combine(Directory.GetCurrentDirectory(), TestDestFolder);
ArtifactDownloadParameters downloadParameters = new ArtifactDownloadParameters();
downloadParameters.TargetDirectory = destPath;
downloadParameters.MinimatchFilters = new string[] { "**" };
BuildArtifact buildArtifact = new BuildArtifact();
buildArtifact.Name = "drop";
buildArtifact.Resource = new ArtifactResource();
buildArtifact.Resource.Data = sourcePath;
await provider.DownloadMultipleArtifactsAsync(downloadParameters, new List <BuildArtifact> {
buildArtifact
}, CancellationToken.None, context);
var sourceFiles = Directory.GetFiles(sourcePath);
var destFiles = Directory.GetFiles(destPath);
Assert.Equal(sourceFiles.Length, destFiles.Length);
foreach (var file in sourceFiles)
{
string destFile = destFiles.FirstOrDefault(f => Path.GetFileName(f).Equals(Path.GetFileName(file)));
Assert.True(StructuralComparisons.StructuralEqualityComparer.Equals(ComputeHash(file), ComputeHash(destFile)));
}
TestCleanup();
}
}
public virtual async Task <BuildArtifact> AssociateArtifact(
int buildId,
string name,
string type,
string data,
Dictionary <string, string> propertiesDictionary,
CancellationToken cancellationToken = default(CancellationToken))
{
BuildArtifact artifact = new BuildArtifact()
{
Name = name,
Resource = new ArtifactResource()
{
Data = data,
Type = type,
Properties = propertiesDictionary
}
};
return(await _buildHttpClient.CreateArtifactAsync(artifact, _projectId, buildId, cancellationToken : cancellationToken));
}
public IArtifactProvider GetProvider(BuildArtifact buildArtifact)
{
string artifactType = buildArtifact.Resource.Type;
if (PipelineArtifactConstants.PipelineArtifact.Equals(artifactType, StringComparison.CurrentCultureIgnoreCase))
{
return(pipelineArtifactProvider);
}
else if (PipelineArtifactConstants.Container.Equals(artifactType, StringComparison.CurrentCultureIgnoreCase))
{
return(fileContainerProvider);
}
else if (PipelineArtifactConstants.FileShareArtifact.Equals(artifactType, StringComparison.CurrentCultureIgnoreCase))
{
return(fileShareProvider);
}
else
{
throw new InvalidOperationException($"{buildArtifact} is not of type PipelineArtifact, FileShare or BuildArtifact");
}
}
private async Task <string> SaveArtifactAsync(BuildArtifact artifact, CancellationToken stoppingToken)
{
var tempFile = Path.GetTempFileName();
// Save the file to disk
using (var tempFileStream = new FileStream(tempFile, FileMode.Create, FileAccess.ReadWrite, FileShare.None))
{
_logger.LogDebug(new EventId(0, "FetchingArtifact"), "Fetching artifact {ArtifactName} from {ArtifactUrl}.", artifact.Name, artifact.Resource.DownloadUrl);
using var response = await _client.GetAsync(artifact.Resource.DownloadUrl, stoppingToken);
if (!response.IsSuccessStatusCode)
{
_logger.LogWarning("Unable to download artifact: {ArtifactName}. Response: {StatusCode}.", artifact.Name, response.StatusCode);
}
var stream = await response.Content.ReadAsStreamAsync();
await stream.CopyToAsync(tempFileStream, stoppingToken);
_logger.LogDebug(new EventId(0, "FetchedArtifact"), "Fetched artifact {ArtifactName} to {TempPath}", artifact.Name, tempFile);
}
return(tempFile);
}
public void AssociateArtifactTest()
{
var returnArtifact = new BuildArtifact();
var token = new CancellationToken();
var guid = Guid.NewGuid();
var service = new BuildService(_mockClientFactory.Object, guid);
_mockBuildClient.Setup(x => x.CreateArtifactAsync(
It.Is <BuildArtifact>(a => a.Name == "name" && a.Resource.Data == "data" && a.Resource.Type == "type" && a.Resource.Properties["key"] == "value"),
It.Is <Guid>(b => b == guid),
It.Is <int>(c => c == 1234),
It.IsAny <object>(),
It.Is <CancellationToken>(d => d == token)))
.Returns(Task.FromResult(returnArtifact));
var artifact = service.AssociateArtifact(1234, "name", "type", "data", new Dictionary <string, string>()
{
{ "key", "value" }
}, token).Result;
Assert.AreEqual(artifact, returnArtifact);
}
public async Task TestDownloadArtifactAsync()
{
byte[] sourceContent1 = GenerateRandomData();
byte[] sourceContent2 = GenerateRandomData();
TestFile sourceFile1 = new TestFile(sourceContent1);
TestFile sourceFile2 = new TestFile(sourceContent2);
sourceFile1.PlaceItem(Path.Combine(Directory.GetCurrentDirectory(), Path.Combine(TestDownloadSourceFolder, "drop/test2.txt")));
sourceFile2.PlaceItem(Path.Combine(Directory.GetCurrentDirectory(), Path.Combine(TestDownloadSourceFolder, "drop/test3.txt")));
using (var hostContext = new TestHostContext(this))
{
var context = new AgentTaskPluginExecutionContext(hostContext.GetTrace());
var provider = new FileShareProvider(context, new CallbackAppTraceSource(str => context.Output(str), System.Diagnostics.SourceLevels.Information));
string sourcePath = Path.Combine(Directory.GetCurrentDirectory(), TestDownloadSourceFolder);
string destPath = Path.Combine(Directory.GetCurrentDirectory(), TestDestFolder);
PipelineArtifactDownloadParameters downloadParameters = new PipelineArtifactDownloadParameters();
downloadParameters.TargetDirectory = destPath;
downloadParameters.MinimatchFilters = new string[] { "**" };
BuildArtifact buildArtifact = new BuildArtifact();
buildArtifact.Name = "drop";
buildArtifact.Resource = new ArtifactResource();
buildArtifact.Resource.Data = sourcePath;
await provider.DownloadSingleArtifactAsync(downloadParameters, buildArtifact, CancellationToken.None);
var sourceFiles = Directory.GetFiles(sourcePath);
var destFiles = Directory.GetFiles(destPath);
Assert.Equal(sourceFiles.Length, destFiles.Length);
foreach (var file in sourceFiles)
{
string destFile = destFiles.FirstOrDefault(f => Path.GetFileName(f).Equals(Path.GetFileName(file)));
Assert.True(StructuralComparisons.StructuralEqualityComparer.Equals(ComputeHash(file), ComputeHash(destFile)));
}
TestCleanup();
}
}
private Task DownloadPipelineArtifact(
BuildDropManager buildDropManager,
BuildArtifact buildArtifact,
string targetDirectory,
string[] minimatchFilters,
CancellationToken cancellationToken)
{
if (buildArtifact.Resource.Type != PipelineArtifactTypeName)
{
throw new ArgumentException("The artifact is not of the type Pipeline Artifact.");
}
var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data);
// 2) download to the target path
DownloadPipelineArtifactOptions options = DownloadPipelineArtifactOptions.CreateWithManifestId(
manifestId,
targetDirectory,
proxyUri: null,
minimatchPatterns: minimatchFilters);
return(buildDropManager.DownloadAsync(options, cancellationToken));
}
// Associate the specified artifact with a build, along with custom data.
public async Task<BuildArtifact> AssociateArtifactAsync(
Guid projectId,
int pipelineId,
string name,
string jobId,
string type,
string data,
Dictionary<string, string> propertiesDictionary,
CancellationToken cancellationToken = default(CancellationToken))
{
BuildArtifact artifact = new BuildArtifact()
{
Name = name,
Source = jobId,
Resource = new ArtifactResource()
{
Data = data,
Type = type,
Properties = propertiesDictionary
}
};
return await _buildHttpClient.CreateArtifactAsync(artifact, projectId, pipelineId, cancellationToken: cancellationToken);
}
// Download pipeline artifact from VSTS BlobStore service through BuildDropManager to a target path
internal async Task DownloadAsync(
AgentTaskPluginExecutionContext context,
Guid projectId,
int buildId,
string artifactName,
string targetDir,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
// 1) get manifest id from artifact data
BuildServer buildHelper = new BuildServer(connection);
BuildArtifact art = await buildHelper.GetArtifact(projectId, buildId, artifactName, cancellationToken);
if (art.Resource.Type != "PipelineArtifact")
{
throw new ArgumentException("The artifact is not of the type Pipeline Artifact\n");
}
var manifestId = DedupIdentifier.Create(art.Resource.Data);
// 2) download to the target path
var buildDropManager = this.GetBDM(context, connection);
await buildDropManager.DownloadAsync(manifestId, targetDir, cancellationToken);
}
private static async Task <string> DownloadArtifactsAsync(BuildHttpClient buildClient, Build build, BuildArtifact artifact, CancellationToken cancellationToken)
{
var tempDirectory = Path.Combine(Path.GetTempPath(), string.Concat(Options.InsertionName, Options.BranchName).Replace(" ", "_").Replace("/", "_"));
if (Directory.Exists(tempDirectory))
{
// Be judicious and clean up old artifacts so we do not eat up memory on the scheduler machine.
Directory.Delete(tempDirectory, recursive: true);
// Sometimes a creation of a directory races with deletion since at least in .net 4.6 deletion is not a blocking call.
// Hence explictly waiting for the directory to be deleted before moving on.
Stopwatch w = Stopwatch.StartNew();
while (Directory.Exists(tempDirectory) && w.ElapsedMilliseconds < 20 * 1000)
{
Thread.Sleep(100);
}
}
Directory.CreateDirectory(tempDirectory);
var archiveDownloadPath = Path.Combine(tempDirectory, string.Concat(artifact.Name, ".zip"));
Log.Trace($"Downloading artifacts to {archiveDownloadPath}");
Stopwatch watch = Stopwatch.StartNew();
using (Stream s = await buildClient.GetArtifactContentZipAsync(Options.TFSProjectName, build.Id, artifact.Name, cancellationToken))
{
using (var fs = File.OpenWrite(archiveDownloadPath))
{
// Using the default buffer size.
await s.CopyToAsync(fs, 81920, cancellationToken);
}
ZipFile.ExtractToDirectory(archiveDownloadPath, tempDirectory);
File.Delete(archiveDownloadPath);
}
Log.Info($"Artifact download took {watch.ElapsedMilliseconds/1000} seconds");
return(Path.Combine(tempDirectory, artifact.Name));
}
public async Task DownloadSingleArtifactAsync(PipelineArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken)
{
await DownloadMultipleArtifactsAsync(downloadParameters, new List <BuildArtifact> {
buildArtifact
}, cancellationToken);
}
private async Task DownloadFileContainerAsync(Guid projectId, BuildArtifact artifact, string rootPath, IEnumerable <string> minimatchPatterns, CancellationToken cancellationToken, bool isSingleArtifactDownload = true)
{
var containerIdAndRoot = ParseContainerId(artifact.Resource.Data);
var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, containerIdAndRoot.Item2);
tracer.Info($"Start downloading FCS artifact- {artifact.Name}");
IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer);
if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0)
{
items = this.GetFilteredItems(items, minimatcherFuncs);
}
if (!isSingleArtifactDownload && items.Any())
{
Directory.CreateDirectory(rootPath);
}
var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder);
Parallel.ForEach(folderItems, (folder) =>
{
var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2);
Directory.CreateDirectory(targetPath);
});
var fileItems = items.Where(i => i.ItemType == ContainerItemType.File);
var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>(
async item =>
{
var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2);
var directory = Path.GetDirectoryName(targetPath);
Directory.CreateDirectory(directory);
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken))
{
tracer.Info($"Downloading: {targetPath}");
using (var targetStream = new FileStream(targetPath, FileMode.Create))
{
await sourceStream.CopyToAsync(targetStream);
}
}
},
maxRetries: 5,
cancellationToken: cancellationToken,
tracer: tracer,
continueOnCapturedContext: false,
canRetryDelegate: exception => exception is IOException,
context: null
);
},
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 5000,
MaxDegreeOfParallelism = 8,
CancellationToken = cancellationToken,
});
await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken);
}
public async Task DownloadSingleArtifactAsync(PipelineArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken)
{
await this.DownloadFileContainerAsync(downloadParameters.ProjectId, buildArtifact, downloadParameters.TargetDirectory, downloadParameters.MinimatchFilters, cancellationToken);
}
public async Task DownloadSingleArtifactAsync(PipelineArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken)
{
var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data);
var options = DownloadDedupManifestArtifactOptions.CreateWithManifestId(
manifestId,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters);
await dedupManifestArtifactClient.DownloadAsync(options, cancellationToken);
}
private async Task DownloadFileContainerAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true)
{
var containerIdAndRoot = ParseContainerId(artifact.Resource.Data);
var projectId = downloadParameters.ProjectId;
var minimatchPatterns = downloadParameters.MinimatchFilters;
var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, isShallow : false, includeBlobMetadata : true, containerIdAndRoot.Item2);
tracer.Info($"Start downloading FCS artifact- {artifact.Name}");
IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer);
if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0)
{
items = this.GetFilteredItems(items, minimatcherFuncs);
}
if (!isSingleArtifactDownload && items.Any())
{
Directory.CreateDirectory(rootPath);
}
var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder);
Parallel.ForEach(folderItems, (folder) =>
{
var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
Directory.CreateDirectory(targetPath);
});
var fileItems = items.Where(i => i.ItemType == ContainerItemType.File);
var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>(
async item =>
{
var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
var directory = Path.GetDirectoryName(targetPath);
Directory.CreateDirectory(directory);
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
tracer.Info($"Downloading: {targetPath}");
if (item.BlobMetadata != null)
{
await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, cancellationToken);
}
else
{
using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken))
using (var targetStream = new FileStream(targetPath, FileMode.Create))
{
await sourceStream.CopyToAsync(targetStream);
}
}
},
maxRetries: downloadParameters.RetryDownloadCount,
cancellationToken: cancellationToken,
tracer: tracer,
continueOnCapturedContext: false,
canRetryDelegate: exception => exception is IOException,
context: null
);
},
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 5000,
MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit,
CancellationToken = cancellationToken,
});
await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken);
// check files (will throw an exception if a file is corrupt)
if (downloadParameters.CheckDownloadedFiles)
{
CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
}
}
public async Task DownloadSingleArtifactAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context)
{
await this.DownloadFileContainerAsync(downloadParameters, buildArtifact, downloadParameters.TargetDirectory, context, cancellationToken);
}
public async Task DownloadSingleArtifactAsync(PipelineArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context)
{
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClient(
this.context, this.connection, cancellationToken, out BlobStoreClientTelemetry clientTelemetry);
using (clientTelemetry) {
var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data);
var options = DownloadDedupManifestArtifactOptions.CreateWithManifestId(
manifestId,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters);
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true,
context: nameof(DownloadSingleArtifactAsync),
cancellationToken: cancellationToken,
continueOnCapturedContext: false);
});
// Send results to CustomerIntelligence
this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
}
public async Task DownloadSingleArtifactAsync(PipelineArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context)
{
await DownloadMultipleArtifactsAsync(downloadParameters, new List <BuildArtifact> {
buildArtifact
}, cancellationToken, context);
}
