public async Task DownloadMultipleArtifactsAsync(PipelineArtifactDownloadParameters downloadParameters, IEnumerable <BuildArtifact> buildArtifacts, CancellationToken cancellationToken)
{
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(
this.context, this.connection, cancellationToken, out BlobStoreClientTelemetry clientTelemetry);
using (clientTelemetry) {
var artifactNameAndManifestIds = buildArtifacts.ToDictionary(
keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong
elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data));
// 2) download to the target path
var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds(
artifactNameAndManifestIds,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters,
minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName);
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
});
// Send results to CustomerIntelligence
this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
}
public async Task DownloadSingleArtifactAsync(PipelineArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken)
{
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(
this.context, this.connection, cancellationToken, out BlobStoreClientTelemetry clientTelemetry);
using (clientTelemetry) {
var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data);
var options = DownloadDedupManifestArtifactOptions.CreateWithManifestId(
manifestId,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters);
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
});
// Send results to CustomerIntelligence
this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
}
public async Task PublishArtifactAsync(
string sourcePath,
string destPath,
int parallelCount,
CancellationToken cancellationToken)
{
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = this.factory.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry);
using (clientTelemetry)
{
FileShareActionRecord publishRecord = clientTelemetry.CreateRecord <FileShareActionRecord>((level, uri, type) =>
new FileShareActionRecord(level, uri, type, nameof(PublishArtifactAsync), context));
await clientTelemetry.MeasureActionAsync(
record : publishRecord,
actionAsync : async() =>
{
return(await PublishArtifactUsingRobocopyAsync(this.context, sourcePath, destPath, parallelCount, cancellationToken));
}
);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: publishRecord);
}
}
internal async Task UploadAsync(
AgentTaskPluginExecutionContext context,
IEnumerable <string> key,
string path,
string salt,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection);
var result = await dedupManifestClient.PublishAsync(path, cancellationToken);
var scope = "myscope";
CreatePipelineCacheArtifactOptions options = new CreatePipelineCacheArtifactOptions
{
Key = key,
RootId = result.RootId,
ManifestId = result.ManifestId,
Scope = scope,
ProofNodes = result.ProofNodes.ToArray(),
Salt = salt
};
var pipelineCacheClient = this.CreateClient(context, connection);
await pipelineCacheClient.CreatePipelineCacheArtifactAsync(options, cancellationToken);
Console.WriteLine("Saved item.");
}
private async Task DownloadPipelineCacheAsync(
AgentTaskPluginExecutionContext context,
DedupManifestArtifactClient dedupManifestClient,
DedupIdentifier manifestId,
string targetDirectory,
string contentFormat,
CancellationToken cancellationToken)
{
if (contentFormat == ContentFormatConstants.SingleTar)
{
string manifestPath = Path.Combine(Path.GetTempPath(), $"{nameof(DedupManifestArtifactClient)}.{Path.GetRandomFileName()}.manifest");
await dedupManifestClient.DownloadFileToPathAsync(manifestId, manifestPath, proxyUri : null, cancellationToken : cancellationToken);
Manifest manifest = JsonSerializer.Deserialize <Manifest>(File.ReadAllText(manifestPath));
await TarUtils.DownloadAndExtractTarAsync(context, manifest, dedupManifestClient, targetDirectory, cancellationToken);
try
{
if (File.Exists(manifestPath))
{
File.Delete(manifestPath);
}
}
catch {}
}
else
{
DownloadDedupManifestArtifactOptions options = DownloadDedupManifestArtifactOptions.CreateWithManifestId(
manifestId,
targetDirectory,
proxyUri: null,
minimatchPatterns: null);
await dedupManifestClient.DownloadAsync(options, cancellationToken);
}
}
internal async Task DownloadAsync(
AgentTaskPluginExecutionContext context,
IEnumerable <string> key,
string path,
string salt,
string variableToSetOnHit,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
var pipelineCacheClient = this.CreateClient(context, connection);
GetPipelineCacheArtifactOptions options = new GetPipelineCacheArtifactOptions
{
Key = key,
Scope = "myscope",
Salt = salt,
};
var result = await pipelineCacheClient.GetPipelineCacheArtifactAsync(options, cancellationToken);
if (result == null)
{
return;
}
else
{
Console.WriteLine("Manifest ID is: {0}", result.ManifestId.ValueString);
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection);
await this.DownloadPipelineCacheAsync(dedupManifestClient, result.ManifestId, path, cancellationToken);
context.SetVariable($"{PipelineCacheVarPrefix}.{variableToSetOnHit}", "True");
Console.WriteLine("Cache restored.");
}
}
private async Task DownloadPipelineCacheAsync(
AgentTaskPluginExecutionContext context,
DedupManifestArtifactClient dedupManifestClient,
DedupIdentifier manifestId,
string[] pathSegments,
string workspaceRoot,
ContentFormat contentFormat,
CancellationToken cancellationToken)
{
if (contentFormat == ContentFormat.SingleTar)
{
string manifestPath = Path.Combine(Path.GetTempPath(), $"{nameof(DedupManifestArtifactClient)}.{Path.GetRandomFileName()}.manifest");
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
await dedupManifestClient.DownloadFileToPathAsync(manifestId, manifestPath, proxyUri: null, cancellationToken: cancellationToken);
},
maxRetries : 3,
tracer : tracer,
canRetryDelegate : e => true,
context : nameof(DownloadPipelineCacheAsync),
cancellationToken : cancellationToken,
continueOnCapturedContext : false);
Manifest manifest = JsonSerializer.Deserialize <Manifest>(File.ReadAllText(manifestPath));
var(tarWorkingDirectory, _) = GetTarWorkingDirectory(pathSegments, workspaceRoot);
await TarUtils.DownloadAndExtractTarAsync(context, manifest, dedupManifestClient, tarWorkingDirectory, cancellationToken);
try
{
if (File.Exists(manifestPath))
{
File.Delete(manifestPath);
}
}
catch { }
}
else
{
DownloadDedupManifestArtifactOptions options = DownloadDedupManifestArtifactOptions.CreateWithManifestId(
manifestId,
pathSegments[0],
proxyUri: null,
minimatchPatterns: null);
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
},
maxRetries : 3,
tracer : tracer,
canRetryDelegate : e => true,
context : nameof(DownloadPipelineCacheAsync),
cancellationToken : cancellationToken,
continueOnCapturedContext : false);
}
}
// Upload from target path to Azure DevOps BlobStore service through DedupManifestArtifactClient, then associate it with the build
internal async Task UploadAsync(
AgentTaskPluginExecutionContext context,
Guid projectId,
int pipelineId,
string name,
string source,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry);
using (clientTelemetry)
{
//Upload the pipeline artifact.
PipelineArtifactActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(UploadAsync), context));
PublishResult result = await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() =>
{
return(await dedupManifestClient.PublishAsync(source, cancellationToken));
}
);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: uploadRecord);
// 2) associate the pipeline artifact with an build artifact
BuildServer buildHelper = new BuildServer(connection);
Dictionary <string, string> propertiesDictionary = new Dictionary <string, string>();
propertiesDictionary.Add(PipelineArtifactConstants.RootId, result.RootId.ValueString);
propertiesDictionary.Add(PipelineArtifactConstants.ProofNodes, StringUtil.ConvertToJson(result.ProofNodes.ToArray()));
propertiesDictionary.Add(PipelineArtifactConstants.ArtifactSize, result.ContentSize.ToString());
BuildArtifact buildArtifact = await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
return(await buildHelper.AssociateArtifactAsync(projectId,
pipelineId,
name,
context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.JobId)?.Value ?? string.Empty,
ArtifactResourceTypes.PipelineArtifact,
result.ManifestId.ValueString,
propertiesDictionary,
cancellationToken));
},
maxRetries : 3,
tracer : tracer,
canRetryDelegate : e => e is TimeoutException || e.InnerException is TimeoutException,
cancellationToken : cancellationToken,
continueOnCapturedContext : false);
context.Output(StringUtil.Loc("AssociateArtifactWithBuild", buildArtifact.Id, pipelineId));
}
}
internal async Task UploadAsync(
AgentTaskPluginExecutionContext context,
Fingerprint fingerprint,
string path,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry);
PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection);
using (clientTelemetry)
{
// Check if the key exists.
PipelineCacheActionRecord cacheRecordGet = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context));
PipelineCacheArtifact getResult = await pipelineCacheClient.GetPipelineCacheArtifactAsync(new [] { fingerprint }, cancellationToken, cacheRecordGet);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecordGet);
//If cache exists, return.
if (getResult != null)
{
context.Output($"Cache with fingerprint `{getResult.Fingerprint}` already exists.");
return;
}
//Upload the pipeline artifact.
PipelineCacheActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, nameof(dedupManifestClient.PublishAsync), context));
PublishResult result = await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() =>
{
return(await dedupManifestClient.PublishAsync(path, cancellationToken));
});
CreatePipelineCacheArtifactOptions options = new CreatePipelineCacheArtifactOptions
{
Fingerprint = fingerprint,
RootId = result.RootId,
ManifestId = result.ManifestId,
ProofNodes = result.ProofNodes.ToArray(),
};
// Cache the artifact
PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.SaveCache, context));
CreateStatus status = await pipelineCacheClient.CreatePipelineCacheArtifactAsync(options, cancellationToken, cacheRecord);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: uploadRecord);
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord);
context.Output("Saved item.");
}
}
internal async Task DownloadAsync(
AgentTaskPluginExecutionContext context,
IEnumerable <string> key,
string path,
string salt,
string variableToSetOnHit,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, out clientTelemetry);
PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection);
GetPipelineCacheArtifactOptions options = new GetPipelineCacheArtifactOptions
{
Key = key,
Salt = salt,
};
using (clientTelemetry)
{
PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context));
PipelineCacheArtifact result = await pipelineCacheClient.GetPipelineCacheArtifactAsync(options, cancellationToken, cacheRecord);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord);
if (result == null)
{
return;
}
else
{
context.Output($"Manifest ID is: {result.ManifestId.ValueString}");
PipelineCacheActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, nameof(DownloadAsync), context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await this.DownloadPipelineCacheAsync(dedupManifestClient, result.ManifestId, path, cancellationToken);
});
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: downloadRecord);
if (!string.IsNullOrEmpty(variableToSetOnHit))
{
context.SetVariable($"{PipelineArtifactConstants.PipelineCache}.{variableToSetOnHit}", "True");
}
Console.WriteLine("Cache restored.");
}
}
}
private DedupManifestArtifactClient CreateDedupManifestClient(AgentTaskPluginExecutionContext context, VssConnection connection)
{
var dedupStoreHttpClient = connection.GetClient <DedupStoreHttpClient>();
var tracer = this.CreateTracer(context);
dedupStoreHttpClient.SetTracer(tracer);
var client = new DedupStoreClientWithDataport(dedupStoreHttpClient, PipelineArtifactProvider.GetDedupStoreClientMaxParallelism(context));
var buildDropManager = new DedupManifestArtifactClient(client, tracer);
return(buildDropManager);
}
public static DedupManifestArtifactClient CreateDedupManifestClient(AgentTaskPluginExecutionContext context, VssConnection connection)
{
var dedupStoreHttpClient = connection.GetClient <DedupStoreHttpClient>();
var tracer = new CallbackAppTraceSource(str => context.Output(str), SourceLevels.Information);
dedupStoreHttpClient.SetTracer(tracer);
var client = new DedupStoreClientWithDataport(dedupStoreHttpClient, DedupStoreClientMaxParallelism);
var dedupManifestClient = new DedupManifestArtifactClient(client, tracer);
return(dedupManifestClient);
}
public PipelineArtifactProvider(AgentTaskPluginExecutionContext context, VssConnection connection, CallbackAppTraceSource tracer)
{
var dedupStoreHttpClient = connection.GetClient <DedupStoreHttpClient>();
this.tracer = tracer;
dedupStoreHttpClient.SetTracer(tracer);
int parallelism = GetDedupStoreClientMaxParallelism(context);
var client = new DedupStoreClientWithDataport(dedupStoreHttpClient, parallelism);
dedupManifestArtifactClient = new DedupManifestArtifactClient(client, this.tracer);
}
private Task DownloadPipelineCacheAsync(
DedupManifestArtifactClient dedupManifestClient,
DedupIdentifier manifestId,
string targetDirectory,
CancellationToken cancellationToken)
{
DownloadPipelineArtifactOptions options = DownloadPipelineArtifactOptions.CreateWithManifestId(
manifestId,
targetDirectory,
proxyUri: null,
minimatchPatterns: null);
return(dedupManifestClient.DownloadAsync(options, cancellationToken));
}
/// <summary>
/// This will download the dedup into stdin stream while extracting the TAR simulataneously (piped). This is done by
/// starting the download through a Task and starting the TAR/7z process which is reading from STDIN.
/// </summary>
/// <remarks>
/// Windows will use 7z to extract the TAR file (only if 7z is installed on the machine and is part of PATH variables).
/// Non-Windows machines will extract TAR file using the 'tar' command'.
/// </remarks>
public static Task DownloadAndExtractTarAsync(
AgentTaskPluginExecutionContext context,
Manifest manifest,
DedupManifestArtifactClient dedupManifestClient,
string tarWorkingDirectory,
CancellationToken cancellationToken)
{
ValidateTarManifest(manifest);
DedupIdentifier dedupId = DedupIdentifier.Create(manifest.Items.Single(i => i.Path.EndsWith(archive, StringComparison.OrdinalIgnoreCase)).Blob.Id);
// We now can simply specify the working directory as the tarball will contain paths relative to it
ProcessStartInfo processStartInfo = GetExtractStartProcessInfo(context, tarWorkingDirectory);
if (!Directory.Exists(tarWorkingDirectory))
{
Directory.CreateDirectory(tarWorkingDirectory);
}
Func <Process, CancellationToken, Task> downloadTaskFunc =
(process, ct) =>
Task.Run(async() =>
{
try
{
await dedupManifestClient.DownloadToStreamAsync(dedupId, process.StandardInput.BaseStream, proxyUri: null, cancellationToken: ct);
process.StandardInput.BaseStream.Close();
}
catch (Exception e)
{
try
{
process.Kill();
}
catch { }
ExceptionDispatchInfo.Capture(e).Throw();
}
});
return(RunProcessAsync(
context,
processStartInfo,
downloadTaskFunc,
() => { },
cancellationToken));
}
internal async Task UploadAsync(
AgentTaskPluginExecutionContext context,
IEnumerable <string> key,
string path,
string salt,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, out clientTelemetry);
PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection);
using (clientTelemetry)
{
//Upload the pipeline artifact.
PipelineCacheActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, nameof(dedupManifestClient.PublishAsync), context));
PublishResult result = await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() =>
{
return(await dedupManifestClient.PublishAsync(path, cancellationToken));
});
CreatePipelineCacheArtifactOptions options = new CreatePipelineCacheArtifactOptions
{
Key = key,
RootId = result.RootId,
ManifestId = result.ManifestId,
ProofNodes = result.ProofNodes.ToArray(),
Salt = salt
};
// Cache the artifact
PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.SaveCache, context));
CreateStatus status = await pipelineCacheClient.CreatePipelineCacheArtifactAsync(options, cancellationToken, cacheRecord);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: uploadRecord);
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord);
context.Output("Saved item.");
}
}
// Upload from target path to Azure DevOps BlobStore service through DedupManifestArtifactClient, then associate it with the build
internal async Task UploadAsync(
AgentTaskPluginExecutionContext context,
Guid projectId,
int pipelineId,
string name,
string source,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry);
using (clientTelemetry)
{
//Upload the pipeline artifact.
PipelineArtifactActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(UploadAsync), context));
PublishResult result = await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() =>
{
return(await dedupManifestClient.PublishAsync(source, cancellationToken));
}
);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: uploadRecord);
// 2) associate the pipeline artifact with an build artifact
BuildServer buildHelper = new BuildServer(connection);
Dictionary <string, string> propertiesDictionary = new Dictionary <string, string>();
propertiesDictionary.Add(PipelineArtifactConstants.RootId, result.RootId.ValueString);
propertiesDictionary.Add(PipelineArtifactConstants.ProofNodes, StringUtil.ConvertToJson(result.ProofNodes.ToArray()));
propertiesDictionary.Add(PipelineArtifactConstants.ArtifactSize, result.ContentSize.ToString());
var artifact = await buildHelper.AssociateArtifactAsync(projectId, pipelineId, name, ArtifactResourceTypes.PipelineArtifact, result.ManifestId.ValueString, propertiesDictionary, cancellationToken);
context.Output(StringUtil.Loc("AssociateArtifactWithBuild", artifact.Id, pipelineId));
}
}
/// <summary>
/// This will download the dedup into stdin stream while extracting the TAR simulataneously (piped). This is done by
/// starting the download through a Task and starting the TAR/7z process which is reading from STDIN.
/// </summary>
/// <remarks>
/// Windows will use 7z to extract the TAR file (only if 7z is installed on the machine and is part of PATH variables).
/// Non-Windows machines will extract TAR file using the 'tar' command'.
/// </remarks>
public static Task DownloadAndExtractTarAsync(
AgentTaskPluginExecutionContext context,
Manifest manifest,
DedupManifestArtifactClient dedupManifestClient,
string targetDirectory,
CancellationToken cancellationToken)
{
ValidateTarManifest(manifest);
Directory.CreateDirectory(targetDirectory);
DedupIdentifier dedupId = DedupIdentifier.Create(manifest.Items.Single(i => i.Path == $"/{archiveFileName}").Blob.Id);
bool does7zExists = isWindows ? CheckIf7ZExists() : false;
string processFileName = (does7zExists) ? "7z" : "tar";
string processArguments = (does7zExists) ? $"x -si -aoa -o{targetDirectory} -ttar" : $"-xf - -C {targetDirectory}";
Func <Process, CancellationToken, Task> downloadTaskFunc =
(process, ct) =>
Task.Run(async() => {
try
{
await dedupManifestClient.DownloadToStreamAsync(dedupId, process.StandardInput.BaseStream, proxyUri: null, cancellationToken: ct);
process.StandardInput.BaseStream.Close();
}
catch (Exception e)
{
process.Kill();
ExceptionDispatchInfo.Capture(e).Throw();
}
});
return(RunProcessAsync(
context,
processFileName,
processArguments,
downloadTaskFunc,
() => { },
cancellationToken));
}
public async Task DownloadMultipleArtifactsAsync(PipelineArtifactDownloadParameters downloadParameters, IEnumerable <BuildArtifact> buildArtifacts, CancellationToken cancellationToken)
{
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = this.factory.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry);
using (clientTelemetry)
{
FileShareActionRecord downloadRecord = clientTelemetry.CreateRecord <FileShareActionRecord>((level, uri, type) =>
new FileShareActionRecord(level, uri, type, nameof(DownloadArtifactsAsync), context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
return(await DownloadArtifactsAsync(downloadParameters, buildArtifacts, cancellationToken));
}
);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
}
// Download with minimatch patterns, V1.
internal async Task DownloadAsync(
AgentTaskPluginExecutionContext context,
PipelineArtifactDownloadParameters downloadParameters,
DownloadOptions downloadOptions,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, out clientTelemetry);
BuildServer buildHelper = new BuildServer(connection);
using (clientTelemetry)
{
// download all pipeline artifacts if artifact name is missing
if (downloadOptions == DownloadOptions.MultiDownload)
{
List <BuildArtifact> artifacts;
if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId)
{
artifacts = await buildHelper.GetArtifactsAsync(downloadParameters.ProjectId, downloadParameters.PipelineId, cancellationToken);
}
else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName)
{
if (string.IsNullOrEmpty(downloadParameters.ProjectName))
{
throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!");
}
else
{
artifacts = await buildHelper.GetArtifactsWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, cancellationToken);
}
}
else
{
throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!");
}
IEnumerable <BuildArtifact> pipelineArtifacts = artifacts.Where(a => a.Resource.Type == PipelineArtifactConstants.PipelineArtifact);
if (pipelineArtifacts.Count() == 0)
{
throw new ArgumentException("Could not find any pipeline artifacts in the build.");
}
else
{
context.Output(StringUtil.Loc("DownloadingMultiplePipelineArtifacts", pipelineArtifacts.Count()));
var artifactNameAndManifestIds = pipelineArtifacts.ToDictionary(
keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong
elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data));
// 2) download to the target path
var options = DownloadPipelineArtifactOptions.CreateWithMultiManifestIds(
artifactNameAndManifestIds,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters);
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
});
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
}
else if (downloadOptions == DownloadOptions.SingleDownload)
{
// 1) get manifest id from artifact data
BuildArtifact buildArtifact;
if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId)
{
buildArtifact = await buildHelper.GetArtifact(downloadParameters.ProjectId, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken);
}
else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName)
{
if (string.IsNullOrEmpty(downloadParameters.ProjectName))
{
throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!");
}
else
{
buildArtifact = await buildHelper.GetArtifactWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken);
}
}
else
{
throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!");
}
var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data);
var options = DownloadPipelineArtifactOptions.CreateWithManifestId(
manifestId,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters);
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
});
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
else
{
throw new InvalidOperationException($"Invalid {nameof(downloadOptions)}!");
}
}
}
internal async Task UploadAsync(
AgentTaskPluginExecutionContext context,
Fingerprint keyFingerprint,
string[] pathSegments,
string workspaceRoot,
CancellationToken cancellationToken,
ContentFormat contentFormat)
{
VssConnection connection = context.VssConnection;
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry);
PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection);
using (clientTelemetry)
{
// Check if the key exists.
PipelineCacheActionRecord cacheRecordGet = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context));
PipelineCacheArtifact getResult = await pipelineCacheClient.GetPipelineCacheArtifactAsync(new[] { keyFingerprint }, cancellationToken, cacheRecordGet);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecordGet);
//If cache exists, return.
if (getResult != null)
{
context.Output($"Cache with fingerprint `{getResult.Fingerprint}` already exists.");
return;
}
context.Output("Resolving path:");
Fingerprint pathFp = FingerprintCreator.EvaluateToFingerprint(context, workspaceRoot, pathSegments, FingerprintType.Path);
context.Output($"Resolved to: {pathFp}");
string uploadPath = await this.GetUploadPathAsync(contentFormat, context, pathFp, pathSegments, workspaceRoot, cancellationToken);
//Upload the pipeline artifact.
PipelineCacheActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, nameof(dedupManifestClient.PublishAsync), context));
PublishResult result = await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() =>
await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
return(await dedupManifestClient.PublishAsync(uploadPath, cancellationToken));
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
cancellationToken: cancellationToken,
continueOnCapturedContext: false)
);
CreatePipelineCacheArtifactContract options = new CreatePipelineCacheArtifactContract
{
Fingerprint = keyFingerprint,
RootId = result.RootId,
ManifestId = result.ManifestId,
ProofNodes = result.ProofNodes.ToArray(),
ContentFormat = contentFormat.ToString(),
};
// delete archive file if it's tar.
if (contentFormat == ContentFormat.SingleTar)
{
try
{
if (File.Exists(uploadPath))
{
File.Delete(uploadPath);
}
}
catch { }
}
// Cache the artifact
PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.SaveCache, context));
CreateStatus status = await pipelineCacheClient.CreatePipelineCacheArtifactAsync(options, cancellationToken, cacheRecord);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: uploadRecord);
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord);
context.Output("Saved item.");
}
}
internal async Task DownloadAsync(
AgentTaskPluginExecutionContext context,
Fingerprint[] fingerprints,
string path,
string cacheHitVariable,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry);
PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection);
using (clientTelemetry)
{
PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context));
PipelineCacheArtifact result = await pipelineCacheClient.GetPipelineCacheArtifactAsync(fingerprints, cancellationToken, cacheRecord);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord);
if (result != null)
{
context.Output($"Entry found at fingerprint: `{result.Fingerprint.ToString()}`");
context.Verbose($"Manifest ID is: {result.ManifestId.ValueString}");
PipelineCacheActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, nameof(DownloadAsync), context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await this.DownloadPipelineCacheAsync(context, dedupManifestClient, result.ManifestId, path, result.ContentFormat, cancellationToken);
});
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: downloadRecord);
context.Output("Cache restored.");
}
if (!string.IsNullOrEmpty(cacheHitVariable))
{
if (result == null)
{
context.SetVariable(cacheHitVariable, "false");
}
else
{
context.Verbose($"Exact fingerprint: `{result.Fingerprint.ToString()}`");
bool foundExact = false;
foreach (var fingerprint in fingerprints)
{
context.Verbose($"This fingerprint: `{fingerprint.ToString()}`");
if (fingerprint == result.Fingerprint ||
result.Fingerprint.Segments.Length == 1 && result.Fingerprint.Segments.Single() == fingerprint.SummarizeForV1())
{
foundExact = true;
break;
}
}
context.SetVariable(cacheHitVariable, foundExact ? "true" : "inexact");
}
}
}
}
internal async Task UploadAsync(
AgentTaskPluginExecutionContext context,
Fingerprint fingerprint,
string path,
CancellationToken cancellationToken,
string preferredContentFormat = ContentFormatConstants.Files)
{
VssConnection connection = context.VssConnection;
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry);
PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection);
using (clientTelemetry)
{
// Check if the key exists.
PipelineCacheActionRecord cacheRecordGet = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context));
PipelineCacheArtifact getResult = await pipelineCacheClient.GetPipelineCacheArtifactAsync(new [] { fingerprint }, cancellationToken, cacheRecordGet);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecordGet);
//If cache exists, return.
if (getResult != null)
{
context.Output($"Cache with fingerprint `{getResult.Fingerprint}` already exists.");
return;
}
string uploadPath = await this.GetUploadPathAsync(preferredContentFormat, context, path, cancellationToken);
//Upload the pipeline artifact.
PipelineCacheActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, nameof(dedupManifestClient.PublishAsync), context));
PublishResult result = await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() =>
{
return(await dedupManifestClient.PublishAsync(uploadPath, cancellationToken));
});
CreatePipelineCacheArtifactContract options = new CreatePipelineCacheArtifactContract
{
Fingerprint = fingerprint,
RootId = result.RootId,
ManifestId = result.ManifestId,
ProofNodes = result.ProofNodes.ToArray(),
ContentFormat = preferredContentFormat
};
// delete archive file if it's tar.
if (string.Equals(preferredContentFormat, ContentFormatConstants.SingleTar, StringComparison.OrdinalIgnoreCase))
{
try
{
if (File.Exists(uploadPath))
{
File.Delete(uploadPath);
}
}
catch { }
}
// Cache the artifact
PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.SaveCache, context));
CreateStatus status = await pipelineCacheClient.CreatePipelineCacheArtifactAsync(options, cancellationToken, cacheRecord);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: uploadRecord);
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord);
context.Output("Saved item.");
}
}
