public async Task <(DedupManifestArtifactClient client, BlobStoreClientTelemetry telemetry)> CreateDedupManifestClientAsync(AgentTaskPluginExecutionContext context, VssConnection connection, CancellationToken cancellationToken)
{
const int maxRetries = 5;
var tracer = context.CreateArtifactsTracer();
var dedupStoreHttpClient = await AsyncHttpRetryHelper.InvokeAsync(
() =>
{
ArtifactHttpClientFactory factory = new ArtifactHttpClientFactory(
connection.Credentials,
TimeSpan.FromSeconds(50),
tracer,
cancellationToken);
// this is actually a hidden network call to the location service:
return(Task.FromResult(factory.CreateVssHttpClient <IDedupStoreHttpClient, DedupStoreHttpClient>(connection.GetClient <DedupStoreHttpClient>().BaseAddress)));
},
maxRetries : maxRetries,
tracer : tracer,
canRetryDelegate : e => true,
context : nameof(CreateDedupManifestClientAsync),
cancellationToken : cancellationToken,
continueOnCapturedContext : false);
var telemetry = new BlobStoreClientTelemetry(tracer, dedupStoreHttpClient.BaseAddress);
var client = new DedupStoreClientWithDataport(dedupStoreHttpClient, PipelineArtifactProvider.GetDedupStoreClientMaxParallelism(context));
return(new DedupManifestArtifactClient(telemetry, client, tracer), telemetry);
}
// Upload from target path to Azure DevOps BlobStore service through DedupManifestArtifactClient, then associate it with the build
internal async Task UploadAsync(
AgentTaskPluginExecutionContext context,
Guid projectId,
int pipelineId,
string name,
string source,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry);
using (clientTelemetry)
{
//Upload the pipeline artifact.
PipelineArtifactActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(UploadAsync), context));
PublishResult result = await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() =>
{
return(await dedupManifestClient.PublishAsync(source, cancellationToken));
}
);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: uploadRecord);
// 2) associate the pipeline artifact with an build artifact
BuildServer buildHelper = new BuildServer(connection);
Dictionary <string, string> propertiesDictionary = new Dictionary <string, string>();
propertiesDictionary.Add(PipelineArtifactConstants.RootId, result.RootId.ValueString);
propertiesDictionary.Add(PipelineArtifactConstants.ProofNodes, StringUtil.ConvertToJson(result.ProofNodes.ToArray()));
propertiesDictionary.Add(PipelineArtifactConstants.ArtifactSize, result.ContentSize.ToString());
BuildArtifact buildArtifact = await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
return(await buildHelper.AssociateArtifactAsync(projectId,
pipelineId,
name,
context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.JobId)?.Value ?? string.Empty,
ArtifactResourceTypes.PipelineArtifact,
result.ManifestId.ValueString,
propertiesDictionary,
cancellationToken));
},
maxRetries : 3,
tracer : tracer,
canRetryDelegate : e => e is TimeoutException || e.InnerException is TimeoutException,
cancellationToken : cancellationToken,
continueOnCapturedContext : false);
context.Output(StringUtil.Loc("AssociateArtifactWithBuild", buildArtifact.Id, pipelineId));
}
}
private Task <PipelineCacheClient> CreateClientWithRetryAsync(
BlobStoreClientTelemetry blobStoreClientTelemetry,
AgentTaskPluginExecutionContext context,
VssConnection connection,
CancellationToken cancellationToken)
{
// this uses location service so needs http retries.
return(AsyncHttpRetryHelper.InvokeAsync(
async() => await this.CreateClientAsync(blobStoreClientTelemetry, context, connection),
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
cancellationToken: cancellationToken,
continueOnCapturedContext: false));
}
await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() => await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
return(await uploadSession.UploadAsync(rootNode, new Dictionary <DedupIdentifier, string>()
{
[dedupId] = itemPath
}, cancellationToken));
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
cancellationToken: cancellationToken,
continueOnCapturedContext: false)
);
public static async Task <(DedupIdentifier dedupId, ulong length)> UploadToBlobStore(
bool verbose,
string itemPath,
Func <TelemetryInformationLevel, Uri, string, BlobStoreTelemetryRecord> telemetryRecordFactory,
Action <string> traceOutput,
DedupStoreClient dedupClient,
BlobStoreClientTelemetry clientTelemetry,
CancellationToken cancellationToken)
{
// Create chunks and identifier
var chunk = await ChunkerHelper.CreateFromFileAsync(FileSystem.Instance, itemPath, cancellationToken, false);
var rootNode = new DedupNode(new [] { chunk });
// ChunkHelper uses 64k block default size
var dedupId = rootNode.GetDedupIdentifier(HashType.Dedup64K);
// Setup upload session to keep file for at mimimum one day
// Blobs will need to be associated with the server with an ID ref otherwise they will be
// garbage collected after one day
var tracer = DedupManifestArtifactClientFactory.CreateArtifactsTracer(verbose, traceOutput);
var keepUntilRef = new KeepUntilBlobReference(DateTime.UtcNow.AddDays(1));
var uploadSession = dedupClient.CreateUploadSession(keepUntilRef, tracer, FileSystem.Instance);
// Upload the chunks
var uploadRecord = clientTelemetry.CreateRecord <BlobStoreTelemetryRecord>(telemetryRecordFactory);
await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() => await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
await uploadSession.UploadAsync(rootNode, new Dictionary <DedupIdentifier, string>()
{
[dedupId] = itemPath
}, cancellationToken);
return(uploadSession.UploadStatistics);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
cancellationToken: cancellationToken,
continueOnCapturedContext: false)
);
return(dedupId, rootNode.TransitiveContentBytes);
}
private async Task <Guid> GetProjectIdAsync(AgentTaskPluginExecutionContext context, string projectName, CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
var projectClient = connection.GetClient <ProjectHttpClient>();
try
{
TeamProject project = await AsyncHttpRetryHelper.InvokeAsync(
async() => await projectClient.GetProject(projectName),
maxRetries : MaxRetries,
tracer : tracer,
context : "GetProjectByName",
cancellationToken : cancellationToken,
continueOnCapturedContext : false);
return(project.Id);
}
catch (Exception ex)
{
throw new ArgumentException("Get project failed for project: " + projectName, ex);
}
}
private async Task <(DedupIdentifier dedupId, ulong length)> UploadToBlobStore(IAsyncCommandContext context, string itemPath, CancellationToken cancellationToken)
{
// Create chunks and identifier
var chunk = await ChunkerHelper.CreateFromFileAsync(FileSystem.Instance, itemPath, cancellationToken, false);
var rootNode = new DedupNode(new [] { chunk });
var dedupId = rootNode.GetDedupIdentifier(HashType.Dedup64K);
// Setup upload session to keep file for at mimimum one day
var verbose = String.Equals(context.GetVariableValueOrDefault("system.debug"), "true", StringComparison.InvariantCultureIgnoreCase);
var tracer = DedupManifestArtifactClientFactory.CreateArtifactsTracer(verbose, (str) => context.Output(str));
var keepUntulRef = new KeepUntilBlobReference(DateTime.UtcNow.AddDays(1));
var uploadSession = _dedupClient.CreateUploadSession(keepUntulRef, tracer, FileSystem.Instance);
// Upload the chunks
var uploadRecord = _blobTelemetry.CreateRecord <BuildArtifactActionRecord>((level, uri, type) =>
new BuildArtifactActionRecord(level, uri, type, nameof(UploadAsync), context));
await _blobTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() => await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
return(await uploadSession.UploadAsync(rootNode, new Dictionary <DedupIdentifier, string>()
{
[dedupId] = itemPath
}, cancellationToken));
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
cancellationToken: cancellationToken,
continueOnCapturedContext: false)
);
return(dedupId, rootNode.TransitiveContentBytes);
}
internal async Task UploadAsync(
AgentTaskPluginExecutionContext context,
Fingerprint fingerprint,
string path,
CancellationToken cancellationToken,
ContentFormat contentFormat)
{
VssConnection connection = context.VssConnection;
var(dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance
.CreateDedupManifestClientAsync(context.IsSystemDebugTrue(), (str) => context.Output(str), connection, cancellationToken);
PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection);
using (clientTelemetry)
{
// Check if the key exists.
PipelineCacheActionRecord cacheRecordGet = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context));
PipelineCacheArtifact getResult = await pipelineCacheClient.GetPipelineCacheArtifactAsync(new [] { fingerprint }, cancellationToken, cacheRecordGet);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecordGet);
//If cache exists, return.
if (getResult != null)
{
context.Output($"Cache with fingerprint `{getResult.Fingerprint}` already exists.");
return;
}
string uploadPath = await this.GetUploadPathAsync(contentFormat, context, path, cancellationToken);
//Upload the pipeline artifact.
PipelineCacheActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, nameof(dedupManifestClient.PublishAsync), context));
PublishResult result = await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() =>
await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
return(await dedupManifestClient.PublishAsync(uploadPath, cancellationToken));
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
cancellationToken: cancellationToken,
continueOnCapturedContext: false)
);
CreatePipelineCacheArtifactContract options = new CreatePipelineCacheArtifactContract
{
Fingerprint = fingerprint,
RootId = result.RootId,
ManifestId = result.ManifestId,
ProofNodes = result.ProofNodes.ToArray(),
ContentFormat = contentFormat.ToString(),
};
// delete archive file if it's tar.
if (contentFormat == ContentFormat.SingleTar)
{
try
{
if (File.Exists(uploadPath))
{
File.Delete(uploadPath);
}
}
catch { }
}
// Cache the artifact
PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.SaveCache, context));
CreateStatus status = await pipelineCacheClient.CreatePipelineCacheArtifactAsync(options, cancellationToken, cacheRecord);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: uploadRecord);
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord);
context.Output("Saved item.");
}
}
public static async Task <(List <BlobFileInfo> fileDedupIds, ulong length)> UploadBatchToBlobstore(
bool verbose,
IReadOnlyList <string> itemPaths,
Func <TelemetryInformationLevel, Uri, string, BlobStoreTelemetryRecord> telemetryRecordFactory,
Action <string> traceOutput,
DedupStoreClient dedupClient,
BlobStoreClientTelemetry clientTelemetry,
CancellationToken cancellationToken,
bool enableReporting = false)
{
// Create chunks and identifier
traceOutput(StringUtil.Loc("BuildingFileTree"));
var fileNodes = await GenerateHashes(itemPaths, cancellationToken);
var rootNode = CreateNodeToUpload(fileNodes.Where(x => x.Success).Select(y => y.Node));
// If there are multiple paths to one DedupId (duplicate files)
// take the last one
var fileDedupIds = new Dictionary <DedupIdentifier, string>();
foreach (var file in fileNodes.Where(x => x.Success))
{
// ChunkHelper uses 64k block default size
var dedupId = file.Node.GetDedupIdentifier(HashType.Dedup64K);
fileDedupIds[dedupId] = file.Path;
}
// Setup upload session to keep file for at mimimum one day
// Blobs will need to be associated with the server with an ID ref otherwise they will be
// garbage collected after one day
var tracer = DedupManifestArtifactClientFactory.CreateArtifactsTracer(verbose, traceOutput);
var keepUntilRef = new KeepUntilBlobReference(DateTime.UtcNow.AddDays(1));
var uploadSession = dedupClient.CreateUploadSession(keepUntilRef, tracer, FileSystem.Instance);
using (var reportingCancelSrc = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
// Log stats
Task reportingTask = null;
if (enableReporting)
{
reportingTask = StartReportingTask(traceOutput, (long)rootNode.TransitiveContentBytes, uploadSession, reportingCancelSrc);
}
// Upload the chunks
var uploadRecord = clientTelemetry.CreateRecord <BlobStoreTelemetryRecord>(telemetryRecordFactory);
await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() => await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
await uploadSession.UploadAsync(rootNode, fileDedupIds, cancellationToken);
return(uploadSession.UploadStatistics);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
cancellationToken: cancellationToken,
continueOnCapturedContext: false)
);
if (enableReporting)
{
reportingCancelSrc.Cancel();
await reportingTask;
}
}
return(fileNodes, rootNode.TransitiveContentBytes);
}
private async Task <int> GetPipelineIdAsync(
AgentTaskPluginExecutionContext context,
string pipelineDefinition,
string pipelineVersionToDownload,
string project,
string[] tagFilters,
BuildResult resultFilter = BuildResult.Succeeded,
string branchName = null,
CancellationToken cancellationToken = default(CancellationToken))
{
if (String.IsNullOrWhiteSpace(pipelineDefinition))
{
throw new InvalidOperationException(StringUtil.Loc("CannotBeNullOrEmpty", "Pipeline Definition"));
}
VssConnection connection = context.VssConnection;
BuildHttpClient buildHttpClient = connection.GetClient <BuildHttpClient>();
var isDefinitionNum = Int32.TryParse(pipelineDefinition, out int definition);
if (!isDefinitionNum)
{
var definitionReferencesWithName = await AsyncHttpRetryHelper.InvokeAsync(
async() => await buildHttpClient.GetDefinitionsAsync(new Guid(project), pipelineDefinition, cancellationToken: cancellationToken),
maxRetries : MaxRetries,
tracer : tracer,
context : "GetBuildDefinitionReferencesByName",
cancellationToken : cancellationToken,
continueOnCapturedContext : false);
var definitionRef = definitionReferencesWithName.FirstOrDefault();
if (definitionRef == null)
{
throw new ArgumentException(StringUtil.Loc("PipelineDoesNotExist", pipelineDefinition));
}
else
{
definition = definitionRef.Id;
}
}
var definitions = new List <int>()
{
definition
};
List <Build> list;
if (pipelineVersionToDownload == pipelineVersionToDownloadLatest)
{
list = await AsyncHttpRetryHelper.InvokeAsync(
async() => await buildHttpClient.GetBuildsAsync(
project,
definitions,
tagFilters: tagFilters,
queryOrder: BuildQueryOrder.FinishTimeDescending,
resultFilter: resultFilter,
cancellationToken: cancellationToken),
maxRetries : MaxRetries,
tracer : tracer,
context : "GetLatestBuild",
cancellationToken : cancellationToken,
continueOnCapturedContext : false);
}
else if (pipelineVersionToDownload == pipelineVersionToDownloadLatestFromBranch)
{
list = await AsyncHttpRetryHelper.InvokeAsync(
async() => await buildHttpClient.GetBuildsAsync(
project,
definitions,
branchName: branchName,
tagFilters: tagFilters,
queryOrder: BuildQueryOrder.FinishTimeDescending,
resultFilter: resultFilter,
cancellationToken: cancellationToken),
maxRetries : MaxRetries,
tracer : tracer,
context : "GetLatestBuildFromBranch",
cancellationToken : cancellationToken,
continueOnCapturedContext : false);
}
else
{
throw new InvalidOperationException("Unreachable code!");
}
if (list.Count > 0)
{
return(list.First().Id);
}
else
{
throw new ArgumentException(StringUtil.Loc("BuildsDoesNotExist"));
}
}
internal async Task DownloadAsync(
AgentTaskPluginExecutionContext context,
Fingerprint[] fingerprints,
string path,
string cacheHitVariable,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
var(dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance
.CreateDedupManifestClientAsync(context.IsSystemDebugTrue(), (str) => context.Output(str), connection, cancellationToken);
PipelineCacheClient pipelineCacheClient = await this.CreateClientWithRetryAsync(clientTelemetry, context, connection, cancellationToken);
using (clientTelemetry)
{
PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context));
PipelineCacheArtifact result = await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
return(await pipelineCacheClient.GetPipelineCacheArtifactAsync(fingerprints, cancellationToken, cacheRecord));
},
maxRetries : 3,
tracer : tracer,
canRetryDelegate : e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
cancellationToken : cancellationToken,
continueOnCapturedContext : false);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord);
if (result != null)
{
context.Output($"Entry found at fingerprint: `{result.Fingerprint.ToString()}`");
context.Verbose($"Manifest ID is: {result.ManifestId.ValueString}");
PipelineCacheActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, nameof(DownloadAsync), context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await this.DownloadPipelineCacheAsync(context, dedupManifestClient, result.ManifestId, path, Enum.Parse <ContentFormat>(result.ContentFormat), cancellationToken);
});
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: downloadRecord);
context.Output("Cache restored.");
}
if (!string.IsNullOrEmpty(cacheHitVariable))
{
if (result == null)
{
context.SetVariable(cacheHitVariable, "false");
}
else
{
context.Verbose($"Exact fingerprint: `{result.Fingerprint.ToString()}`");
bool foundExact = false;
foreach (var fingerprint in fingerprints)
{
context.Verbose($"This fingerprint: `{fingerprint.ToString()}`");
if (fingerprint == result.Fingerprint ||
result.Fingerprint.Segments.Length == 1 && result.Fingerprint.Segments.Single() == fingerprint.SummarizeForV1())
{
foundExact = true;
break;
}
}
context.SetVariable(cacheHitVariable, foundExact ? "true" : "inexact");
}
}
}
}