// Download pipeline artifact from VSTS BlobStore service through BuildDropManager to a target path
internal async Task DownloadAsync(
AgentTaskPluginExecutionContext context,
Guid projectId,
int buildId,
string artifactName,
string targetDir,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
// 1) get manifest id from artifact data
BuildServer buildHelper = new BuildServer(connection);
BuildArtifact art = await buildHelper.GetArtifact(projectId, buildId, artifactName, cancellationToken);
var manifestId = DedupIdentifier.Create(art.Resource.Data);
// 2) download to the target path
var httpclient = connection.GetClient <DedupStoreHttpClient>();
var tracer = new CallbackAppTraceSource(str => context.Output(str), System.Diagnostics.SourceLevels.Information);
httpclient.SetTracer(tracer);
var client = new DedupStoreClientWithDataport(httpclient, maxParallelism: 16 * Environment.ProcessorCount);
var BuildDropManager = new BuildDropManager(client, tracer);
await BuildDropManager.DownloadAsync(manifestId, targetDir, cancellationToken);
}
// Upload from target path to VSTS BlobStore service through BuildDropManager, then associate it with the build
internal async Task UploadAsync(
AgentTaskPluginExecutionContext context,
Guid projectId,
int buildId,
string name,
string source,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
// 1) upload pipeline artifact to VSTS BlobStore
var httpclient = connection.GetClient <DedupStoreHttpClient>();
var tracer = new CallbackAppTraceSource(str => context.Output(str), System.Diagnostics.SourceLevels.Information);
httpclient.SetTracer(tracer);
var client = new DedupStoreClientWithDataport(httpclient, 16 * Environment.ProcessorCount);
var BuildDropManager = new BuildDropManager(client, tracer);
var result = await BuildDropManager.PublishAsync(source, cancellationToken);
// 2) associate the pipeline artifact with an build artifact
BuildServer buildHelper = new BuildServer(connection);
Dictionary <string, string> propertiesDictionary = new Dictionary <string, string>();
propertiesDictionary.Add(RootId, result.RootId.ValueString);
propertiesDictionary.Add(ProofNodes, StringUtil.ConvertToJson(result.ProofNodes.ToArray()));
var artifact = await buildHelper.AssociateArtifact(projectId, buildId, name, ArtifactResourceTypes.PipelineArtifact, result.ManifestId.ValueString, propertiesDictionary, cancellationToken);
context.Output(StringUtil.Loc("AssociateArtifactWithBuild", artifact.Id, buildId));
}
public async Task <(DedupManifestArtifactClient client, BlobStoreClientTelemetry telemetry)> CreateDedupManifestClientAsync(AgentTaskPluginExecutionContext context, VssConnection connection, CancellationToken cancellationToken)
{
const int maxRetries = 5;
var tracer = context.CreateArtifactsTracer();
var dedupStoreHttpClient = await AsyncHttpRetryHelper.InvokeAsync(
() =>
{
ArtifactHttpClientFactory factory = new ArtifactHttpClientFactory(
connection.Credentials,
TimeSpan.FromSeconds(50),
tracer,
cancellationToken);
// this is actually a hidden network call to the location service:
return(Task.FromResult(factory.CreateVssHttpClient <IDedupStoreHttpClient, DedupStoreHttpClient>(connection.GetClient <DedupStoreHttpClient>().BaseAddress)));
},
maxRetries : maxRetries,
tracer : tracer,
canRetryDelegate : e => true,
context : nameof(CreateDedupManifestClientAsync),
cancellationToken : cancellationToken,
continueOnCapturedContext : false);
var telemetry = new BlobStoreClientTelemetry(tracer, dedupStoreHttpClient.BaseAddress);
var client = new DedupStoreClientWithDataport(dedupStoreHttpClient, PipelineArtifactProvider.GetDedupStoreClientMaxParallelism(context));
return(new DedupManifestArtifactClient(telemetry, client, tracer), telemetry);
}
public static DedupManifestArtifactClient CreateDedupManifestClient(AgentTaskPluginExecutionContext context, VssConnection connection, out BlobStoreClientTelemetry telemetry)
{
var dedupStoreHttpClient = connection.GetClient <DedupStoreHttpClient>();
var tracer = new CallbackAppTraceSource(str => context.Output(str), SourceLevels.Information);
dedupStoreHttpClient.SetTracer(tracer);
var client = new DedupStoreClientWithDataport(dedupStoreHttpClient, PipelineArtifactProvider.GetDedupStoreClientMaxParallelism(context));
return(new DedupManifestArtifactClient(telemetry = new BlobStoreClientTelemetry(tracer, dedupStoreHttpClient.BaseAddress), client, tracer));
}
public PipelineArtifactProvider(AgentTaskPluginExecutionContext context, VssConnection connection, CallbackAppTraceSource tracer)
{
var dedupStoreHttpClient = connection.GetClient <DedupStoreHttpClient>();
this.tracer = tracer;
dedupStoreHttpClient.SetTracer(tracer);
var client = new DedupStoreClientWithDataport(dedupStoreHttpClient, 16 * Environment.ProcessorCount);
buildDropManager = new BuildDropManager(client, this.tracer);
}
private BuildDropManager CreateBulidDropManager(AgentTaskPluginExecutionContext context, VssConnection connection)
{
var dedupStoreHttpClient = connection.GetClient <DedupStoreHttpClient>();
var tracer = this.CreateTracer(context);
dedupStoreHttpClient.SetTracer(tracer);
var client = new DedupStoreClientWithDataport(dedupStoreHttpClient, 16 * Environment.ProcessorCount);
var buildDropManager = new BuildDropManager(client, tracer);
return(buildDropManager);
}
public PipelineArtifactProvider(AgentTaskPluginExecutionContext context, VssConnection connection, IAppTraceSource tracer)
{
var dedupStoreHttpClient = connection.GetClient <DedupStoreHttpClient>();
this.tracer = tracer;
this.context = context;
this.connection = connection;
dedupStoreHttpClient.SetTracer(tracer);
int parallelism = GetDedupStoreClientMaxParallelism(context);
var client = new DedupStoreClientWithDataport(dedupStoreHttpClient, parallelism);
}
private BuildDropManager CreateBulidDropManager(AgentTaskPluginExecutionContext context, VssConnection connection)
{
var dedupStoreHttpClient = connection.GetClient <DedupStoreHttpClient>();
var tracer = new CallbackAppTraceSource(str => context.Output(str), System.Diagnostics.SourceLevels.Information);
dedupStoreHttpClient.SetTracer(tracer);
var client = new DedupStoreClientWithDataport(dedupStoreHttpClient, 16 * Environment.ProcessorCount);
var buildDropManager = new BuildDropManager(client, tracer);
return(buildDropManager);
}
private BuildDropManager CreateBulidDropManager(AgentTaskPluginExecutionContext context, VssConnection connection)
{
var dedupStoreHttpClient = connection.GetClient <DedupStoreHttpClient>();
var tracer = this.CreateTracer(context);
dedupStoreHttpClient.SetTracer(tracer);
var client = new DedupStoreClientWithDataport(dedupStoreHttpClient, PipelineArtifactProvider.GetDedupStoreClientMaxParallelism(context));
var buildDropManager = new BuildDropManager(client, tracer);
return(buildDropManager);
}
public static DedupManifestArtifactClient CreateDedupManifestClient(AgentTaskPluginExecutionContext context, VssConnection connection)
{
var dedupStoreHttpClient = connection.GetClient <DedupStoreHttpClient>();
var tracer = new CallbackAppTraceSource(str => context.Output(str), SourceLevels.Information);
dedupStoreHttpClient.SetTracer(tracer);
var client = new DedupStoreClientWithDataport(dedupStoreHttpClient, DedupStoreClientMaxParallelism);
var dedupManifestClient = new DedupManifestArtifactClient(client, tracer);
return(dedupManifestClient);
}
public DedupManifestArtifactClient CreateDedupManifestClient(AgentTaskPluginExecutionContext context, VssConnection connection, CancellationToken cancellationToken, out BlobStoreClientTelemetry telemetry)
{
var tracer = context.CreateArtifactsTracer();
ArtifactHttpClientFactory factory = new ArtifactHttpClientFactory(
connection.Credentials,
TimeSpan.FromSeconds(50),
tracer,
cancellationToken);
var dedupStoreHttpClient = factory.CreateVssHttpClient <IDedupStoreHttpClient, DedupStoreHttpClient>(connection.GetClient <DedupStoreHttpClient>().BaseAddress);
var client = new DedupStoreClientWithDataport(dedupStoreHttpClient, PipelineArtifactProvider.GetDedupStoreClientMaxParallelism(context));
return(new DedupManifestArtifactClient(telemetry = new BlobStoreClientTelemetry(tracer, dedupStoreHttpClient.BaseAddress), client, tracer));
}
private async Task <UploadResult> ParallelUploadAsync(IAsyncCommandContext context, IReadOnlyList <string> files, int concurrentUploads, CancellationToken token)
{
// return files that fail to upload and total artifact size
var uploadResult = new UploadResult();
// nothing needs to upload
if (files.Count == 0)
{
return(uploadResult);
}
var uploadToBlob = String.Equals(context.GetVariableValueOrDefault("agent.UploadBuildArtifactsToBlob"), "true", StringComparison.InvariantCultureIgnoreCase);
if (uploadToBlob)
{
var verbose = String.Equals(context.GetVariableValueOrDefault("system.debug"), "true", StringComparison.InvariantCultureIgnoreCase);
var(dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance
.CreateDedupClientAsync(verbose, (str) => context.Output(str), this._connection, token);
_dedupClient = dedupClient;
_blobTelemetry = clientTelemetry;
}
// ensure the file upload queue is empty.
if (!_fileUploadQueue.IsEmpty)
{
throw new ArgumentOutOfRangeException(nameof(_fileUploadQueue));
}
// enqueue file into upload queue.
foreach (var file in files)
{
_fileUploadQueue.Enqueue(file);
}
// Start upload monitor task.
_filesProcessed = 0;
_uploadFinished = new TaskCompletionSource <int>();
_fileUploadTraceLog.Clear();
_fileUploadProgressLog.Clear();
Task uploadMonitor = ReportingAsync(context, files.Count(), _uploadCancellationTokenSource.Token);
// Start parallel upload tasks.
List <Task <UploadResult> > parallelUploadingTasks = new List <Task <UploadResult> >();
for (int uploader = 0; uploader < concurrentUploads; uploader++)
{
parallelUploadingTasks.Add(UploadAsync(context, uploader, uploadToBlob, _uploadCancellationTokenSource.Token));
}
// Wait for parallel upload finish.
await Task.WhenAll(parallelUploadingTasks);
foreach (var uploadTask in parallelUploadingTasks)
{
// record all failed files.
uploadResult.AddUploadResult(await uploadTask);
}
// Stop monitor task;
_uploadFinished.TrySetResult(0);
await uploadMonitor;
return(uploadResult);
}
