public async Task DownloadMultipleArtifactsAsync(PipelineArtifactDownloadParameters downloadParameters, IEnumerable <BuildArtifact> buildArtifacts, CancellationToken cancellationToken)
{
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(
this.context, this.connection, cancellationToken, out BlobStoreClientTelemetry clientTelemetry);
using (clientTelemetry) {
var artifactNameAndManifestIds = buildArtifacts.ToDictionary(
keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong
elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data));
// 2) download to the target path
var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds(
artifactNameAndManifestIds,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters,
minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName);
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
});
// Send results to CustomerIntelligence
this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
}
public async Task DownloadSingleArtifactAsync(PipelineArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken)
{
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(
this.context, this.connection, cancellationToken, out BlobStoreClientTelemetry clientTelemetry);
using (clientTelemetry) {
var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data);
var options = DownloadDedupManifestArtifactOptions.CreateWithManifestId(
manifestId,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters);
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
});
// Send results to CustomerIntelligence
this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
}
// Upload from target path to Azure DevOps BlobStore service through DedupManifestArtifactClient, then associate it with the build
internal async Task UploadAsync(
AgentTaskPluginExecutionContext context,
Guid projectId,
int pipelineId,
string name,
string source,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry);
using (clientTelemetry)
{
//Upload the pipeline artifact.
PipelineArtifactActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(UploadAsync), context));
PublishResult result = await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() =>
{
return(await dedupManifestClient.PublishAsync(source, cancellationToken));
}
);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: uploadRecord);
// 2) associate the pipeline artifact with an build artifact
BuildServer buildHelper = new BuildServer(connection);
Dictionary <string, string> propertiesDictionary = new Dictionary <string, string>();
propertiesDictionary.Add(PipelineArtifactConstants.RootId, result.RootId.ValueString);
propertiesDictionary.Add(PipelineArtifactConstants.ProofNodes, StringUtil.ConvertToJson(result.ProofNodes.ToArray()));
propertiesDictionary.Add(PipelineArtifactConstants.ArtifactSize, result.ContentSize.ToString());
BuildArtifact buildArtifact = await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
return(await buildHelper.AssociateArtifactAsync(projectId,
pipelineId,
name,
context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.JobId)?.Value ?? string.Empty,
ArtifactResourceTypes.PipelineArtifact,
result.ManifestId.ValueString,
propertiesDictionary,
cancellationToken));
},
maxRetries : 3,
tracer : tracer,
canRetryDelegate : e => e is TimeoutException || e.InnerException is TimeoutException,
cancellationToken : cancellationToken,
continueOnCapturedContext : false);
context.Output(StringUtil.Loc("AssociateArtifactWithBuild", buildArtifact.Id, pipelineId));
}
}
public async Task DownloadMultipleArtifactsAsync(ArtifactDownloadParameters downloadParameters, IEnumerable <BuildArtifact> buildArtifacts, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context)
{
var(dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClientAsync(
this.context.IsSystemDebugTrue(), (str) => this.context.Output(str), this.connection, cancellationToken);
using (clientTelemetry) {
var artifactNameAndManifestIds = buildArtifacts.ToDictionary(
keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong
elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data));
// 2) download to the target path
var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds(
artifactNameAndManifestIds,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters,
minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName);
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true,
context: nameof(DownloadMultipleArtifactsAsync),
cancellationToken: cancellationToken,
continueOnCapturedContext: false);
});
// Send results to CustomerIntelligence
this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
}
// Upload from target path to Azure DevOps BlobStore service through DedupManifestArtifactClient, then associate it with the build
internal async Task UploadAsync(
AgentTaskPluginExecutionContext context,
Guid projectId,
int pipelineId,
string name,
string source,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry);
using (clientTelemetry)
{
//Upload the pipeline artifact.
PipelineArtifactActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(UploadAsync), context));
PublishResult result = await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() =>
{
return(await dedupManifestClient.PublishAsync(source, cancellationToken));
}
);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: uploadRecord);
// 2) associate the pipeline artifact with an build artifact
BuildServer buildHelper = new BuildServer(connection);
Dictionary <string, string> propertiesDictionary = new Dictionary <string, string>();
propertiesDictionary.Add(PipelineArtifactConstants.RootId, result.RootId.ValueString);
propertiesDictionary.Add(PipelineArtifactConstants.ProofNodes, StringUtil.ConvertToJson(result.ProofNodes.ToArray()));
propertiesDictionary.Add(PipelineArtifactConstants.ArtifactSize, result.ContentSize.ToString());
var artifact = await buildHelper.AssociateArtifactAsync(projectId, pipelineId, name, ArtifactResourceTypes.PipelineArtifact, result.ManifestId.ValueString, propertiesDictionary, cancellationToken);
context.Output(StringUtil.Loc("AssociateArtifactWithBuild", artifact.Id, pipelineId));
}
}
public async Task DownloadSingleArtifactAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context)
{
var(dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClientAsync(
this.context.IsSystemDebugTrue(), (str) => this.context.Output(str), this.connection, cancellationToken);
using (clientTelemetry) {
var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data);
var options = DownloadDedupManifestArtifactOptions.CreateWithManifestId(
manifestId,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters);
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true,
context: nameof(DownloadSingleArtifactAsync),
cancellationToken: cancellationToken,
continueOnCapturedContext: false);
});
// Send results to CustomerIntelligence
this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
}
// Download with minimatch patterns, V1.
internal async Task DownloadAsync(
AgentTaskPluginExecutionContext context,
PipelineArtifactDownloadParameters downloadParameters,
DownloadOptions downloadOptions,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, out clientTelemetry);
BuildServer buildHelper = new BuildServer(connection);
using (clientTelemetry)
{
// download all pipeline artifacts if artifact name is missing
if (downloadOptions == DownloadOptions.MultiDownload)
{
List <BuildArtifact> artifacts;
if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId)
{
artifacts = await buildHelper.GetArtifactsAsync(downloadParameters.ProjectId, downloadParameters.PipelineId, cancellationToken);
}
else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName)
{
if (string.IsNullOrEmpty(downloadParameters.ProjectName))
{
throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!");
}
else
{
artifacts = await buildHelper.GetArtifactsWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, cancellationToken);
}
}
else
{
throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!");
}
IEnumerable <BuildArtifact> pipelineArtifacts = artifacts.Where(a => a.Resource.Type == PipelineArtifactConstants.PipelineArtifact);
if (pipelineArtifacts.Count() == 0)
{
throw new ArgumentException("Could not find any pipeline artifacts in the build.");
}
else
{
context.Output(StringUtil.Loc("DownloadingMultiplePipelineArtifacts", pipelineArtifacts.Count()));
var artifactNameAndManifestIds = pipelineArtifacts.ToDictionary(
keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong
elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data));
// 2) download to the target path
var options = DownloadPipelineArtifactOptions.CreateWithMultiManifestIds(
artifactNameAndManifestIds,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters);
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
});
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
}
else if (downloadOptions == DownloadOptions.SingleDownload)
{
// 1) get manifest id from artifact data
BuildArtifact buildArtifact;
if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId)
{
buildArtifact = await buildHelper.GetArtifact(downloadParameters.ProjectId, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken);
}
else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName)
{
if (string.IsNullOrEmpty(downloadParameters.ProjectName))
{
throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!");
}
else
{
buildArtifact = await buildHelper.GetArtifactWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken);
}
}
else
{
throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!");
}
var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data);
var options = DownloadPipelineArtifactOptions.CreateWithManifestId(
manifestId,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters);
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
});
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
else
{
throw new InvalidOperationException($"Invalid {nameof(downloadOptions)}!");
}
}
}
