public async Task <(DedupManifestArtifactClient client, BlobStoreClientTelemetry telemetry)> CreateDedupManifestClientAsync(AgentTaskPluginExecutionContext context, VssConnection connection, CancellationToken cancellationToken)
{
const int maxRetries = 5;
var tracer = context.CreateArtifactsTracer();
var dedupStoreHttpClient = await AsyncHttpRetryHelper.InvokeAsync(
() =>
{
ArtifactHttpClientFactory factory = new ArtifactHttpClientFactory(
connection.Credentials,
TimeSpan.FromSeconds(50),
tracer,
cancellationToken);
// this is actually a hidden network call to the location service:
return(Task.FromResult(factory.CreateVssHttpClient <IDedupStoreHttpClient, DedupStoreHttpClient>(connection.GetClient <DedupStoreHttpClient>().BaseAddress)));
},
maxRetries : maxRetries,
tracer : tracer,
canRetryDelegate : e => true,
context : nameof(CreateDedupManifestClientAsync),
cancellationToken : cancellationToken,
continueOnCapturedContext : false);
var telemetry = new BlobStoreClientTelemetry(tracer, dedupStoreHttpClient.BaseAddress);
var client = new DedupStoreClientWithDataport(dedupStoreHttpClient, PipelineArtifactProvider.GetDedupStoreClientMaxParallelism(context));
return(new DedupManifestArtifactClient(telemetry, client, tracer), telemetry);
}
private async Task DownloadPipelineCacheAsync(
AgentTaskPluginExecutionContext context,
DedupManifestArtifactClient dedupManifestClient,
DedupIdentifier manifestId,
string[] pathSegments,
string workspaceRoot,
ContentFormat contentFormat,
CancellationToken cancellationToken)
{
if (contentFormat == ContentFormat.SingleTar)
{
string manifestPath = Path.Combine(Path.GetTempPath(), $"{nameof(DedupManifestArtifactClient)}.{Path.GetRandomFileName()}.manifest");
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
await dedupManifestClient.DownloadFileToPathAsync(manifestId, manifestPath, proxyUri: null, cancellationToken: cancellationToken);
},
maxRetries : 3,
tracer : tracer,
canRetryDelegate : e => true,
context : nameof(DownloadPipelineCacheAsync),
cancellationToken : cancellationToken,
continueOnCapturedContext : false);
Manifest manifest = JsonSerializer.Deserialize <Manifest>(File.ReadAllText(manifestPath));
var(tarWorkingDirectory, _) = GetTarWorkingDirectory(pathSegments, workspaceRoot);
await TarUtils.DownloadAndExtractTarAsync(context, manifest, dedupManifestClient, tarWorkingDirectory, cancellationToken);
try
{
if (File.Exists(manifestPath))
{
File.Delete(manifestPath);
}
}
catch { }
}
else
{
DownloadDedupManifestArtifactOptions options = DownloadDedupManifestArtifactOptions.CreateWithManifestId(
manifestId,
pathSegments[0],
proxyUri: null,
minimatchPatterns: null);
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
},
maxRetries : 3,
tracer : tracer,
canRetryDelegate : e => true,
context : nameof(DownloadPipelineCacheAsync),
cancellationToken : cancellationToken,
continueOnCapturedContext : false);
}
}
// Upload from target path to Azure DevOps BlobStore service through DedupManifestArtifactClient, then associate it with the build
internal async Task UploadAsync(
AgentTaskPluginExecutionContext context,
Guid projectId,
int pipelineId,
string name,
string source,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry);
using (clientTelemetry)
{
//Upload the pipeline artifact.
PipelineArtifactActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(UploadAsync), context));
PublishResult result = await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() =>
{
return(await dedupManifestClient.PublishAsync(source, cancellationToken));
}
);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: uploadRecord);
// 2) associate the pipeline artifact with an build artifact
BuildServer buildHelper = new BuildServer(connection);
Dictionary <string, string> propertiesDictionary = new Dictionary <string, string>();
propertiesDictionary.Add(PipelineArtifactConstants.RootId, result.RootId.ValueString);
propertiesDictionary.Add(PipelineArtifactConstants.ProofNodes, StringUtil.ConvertToJson(result.ProofNodes.ToArray()));
propertiesDictionary.Add(PipelineArtifactConstants.ArtifactSize, result.ContentSize.ToString());
BuildArtifact buildArtifact = await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
return(await buildHelper.AssociateArtifactAsync(projectId,
pipelineId,
name,
context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.JobId)?.Value ?? string.Empty,
ArtifactResourceTypes.PipelineArtifact,
result.ManifestId.ValueString,
propertiesDictionary,
cancellationToken));
},
maxRetries : 3,
tracer : tracer,
canRetryDelegate : e => e is TimeoutException || e.InnerException is TimeoutException,
cancellationToken : cancellationToken,
continueOnCapturedContext : false);
context.Output(StringUtil.Loc("AssociateArtifactWithBuild", buildArtifact.Id, pipelineId));
}
}
private Task <PipelineCacheClient> CreateClientWithRetryAsync(
BlobStoreClientTelemetry blobStoreClientTelemetry,
AgentTaskPluginExecutionContext context,
VssConnection connection,
CancellationToken cancellationToken)
{
// this uses location service so needs http retries.
return(AsyncHttpRetryHelper.InvokeAsync(
async() => await this.CreateClientAsync(blobStoreClientTelemetry, context, connection),
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
cancellationToken: cancellationToken,
continueOnCapturedContext: false));
}
await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() => await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
return(await uploadSession.UploadAsync(rootNode, new Dictionary <DedupIdentifier, string>()
{
[dedupId] = itemPath
}, cancellationToken));
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
cancellationToken: cancellationToken,
continueOnCapturedContext: false)
);
public static async Task <(DedupIdentifier dedupId, ulong length)> UploadToBlobStore(
bool verbose,
string itemPath,
Func <TelemetryInformationLevel, Uri, string, BlobStoreTelemetryRecord> telemetryRecordFactory,
Action <string> traceOutput,
DedupStoreClient dedupClient,
BlobStoreClientTelemetry clientTelemetry,
CancellationToken cancellationToken)
{
// Create chunks and identifier
var chunk = await ChunkerHelper.CreateFromFileAsync(FileSystem.Instance, itemPath, cancellationToken, false);
var rootNode = new DedupNode(new [] { chunk });
// ChunkHelper uses 64k block default size
var dedupId = rootNode.GetDedupIdentifier(HashType.Dedup64K);
// Setup upload session to keep file for at mimimum one day
// Blobs will need to be associated with the server with an ID ref otherwise they will be
// garbage collected after one day
var tracer = DedupManifestArtifactClientFactory.CreateArtifactsTracer(verbose, traceOutput);
var keepUntilRef = new KeepUntilBlobReference(DateTime.UtcNow.AddDays(1));
var uploadSession = dedupClient.CreateUploadSession(keepUntilRef, tracer, FileSystem.Instance);
// Upload the chunks
var uploadRecord = clientTelemetry.CreateRecord <BlobStoreTelemetryRecord>(telemetryRecordFactory);
await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() => await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
await uploadSession.UploadAsync(rootNode, new Dictionary <DedupIdentifier, string>()
{
[dedupId] = itemPath
}, cancellationToken);
return(uploadSession.UploadStatistics);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
cancellationToken: cancellationToken,
continueOnCapturedContext: false)
);
return(dedupId, rootNode.TransitiveContentBytes);
}
public async Task DownloadMultipleArtifactsAsync(PipelineArtifactDownloadParameters downloadParameters, IEnumerable <BuildArtifact> buildArtifacts, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context)
{
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClient(
this.context, this.connection, cancellationToken, out BlobStoreClientTelemetry clientTelemetry);
using (clientTelemetry) {
var artifactNameAndManifestIds = buildArtifacts.ToDictionary(
keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong
elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data));
// 2) download to the target path
var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds(
artifactNameAndManifestIds,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters,
minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName);
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true,
context: nameof(DownloadMultipleArtifactsAsync),
cancellationToken: cancellationToken,
continueOnCapturedContext: false);
});
// Send results to CustomerIntelligence
this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
}
private async Task <Guid> GetProjectIdAsync(AgentTaskPluginExecutionContext context, string projectName, CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
var projectClient = connection.GetClient <ProjectHttpClient>();
try
{
TeamProject project = await AsyncHttpRetryHelper.InvokeAsync(
async() => await projectClient.GetProject(projectName),
maxRetries : MaxRetries,
tracer : tracer,
context : "GetProjectByName",
cancellationToken : cancellationToken,
continueOnCapturedContext : false);
return(project.Id);
}
catch (Exception ex)
{
throw new ArgumentException("Get project failed for project: " + projectName, ex);
}
}
public async Task DownloadSingleArtifactAsync(PipelineArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context)
{
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClient(
this.context, this.connection, cancellationToken, out BlobStoreClientTelemetry clientTelemetry);
using (clientTelemetry) {
var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data);
var options = DownloadDedupManifestArtifactOptions.CreateWithManifestId(
manifestId,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters);
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true,
context: nameof(DownloadSingleArtifactAsync),
cancellationToken: cancellationToken,
continueOnCapturedContext: false);
});
// Send results to CustomerIntelligence
this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
}
private async Task <(DedupIdentifier dedupId, ulong length)> UploadToBlobStore(IAsyncCommandContext context, string itemPath, CancellationToken cancellationToken)
{
// Create chunks and identifier
var chunk = await ChunkerHelper.CreateFromFileAsync(FileSystem.Instance, itemPath, cancellationToken, false);
var rootNode = new DedupNode(new [] { chunk });
var dedupId = rootNode.GetDedupIdentifier(HashType.Dedup64K);
// Setup upload session to keep file for at mimimum one day
var verbose = String.Equals(context.GetVariableValueOrDefault("system.debug"), "true", StringComparison.InvariantCultureIgnoreCase);
var tracer = DedupManifestArtifactClientFactory.CreateArtifactsTracer(verbose, (str) => context.Output(str));
var keepUntulRef = new KeepUntilBlobReference(DateTime.UtcNow.AddDays(1));
var uploadSession = _dedupClient.CreateUploadSession(keepUntulRef, tracer, FileSystem.Instance);
// Upload the chunks
var uploadRecord = _blobTelemetry.CreateRecord <BuildArtifactActionRecord>((level, uri, type) =>
new BuildArtifactActionRecord(level, uri, type, nameof(UploadAsync), context));
await _blobTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() => await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
return(await uploadSession.UploadAsync(rootNode, new Dictionary <DedupIdentifier, string>()
{
[dedupId] = itemPath
}, cancellationToken));
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
cancellationToken: cancellationToken,
continueOnCapturedContext: false)
);
return(dedupId, rootNode.TransitiveContentBytes);
}
private async Task DownloadFileContainerAsync(Guid projectId, BuildArtifact artifact, string rootPath, IEnumerable <string> minimatchPatterns, CancellationToken cancellationToken, bool isSingleArtifactDownload = true)
{
var containerIdAndRoot = ParseContainerId(artifact.Resource.Data);
var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, containerIdAndRoot.Item2);
tracer.Info($"Start downloading FCS artifact- {artifact.Name}");
IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer);
if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0)
{
items = this.GetFilteredItems(items, minimatcherFuncs);
}
if (!isSingleArtifactDownload && items.Any())
{
Directory.CreateDirectory(rootPath);
}
var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder);
Parallel.ForEach(folderItems, (folder) =>
{
var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2);
Directory.CreateDirectory(targetPath);
});
var fileItems = items.Where(i => i.ItemType == ContainerItemType.File);
var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>(
async item =>
{
var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2);
var directory = Path.GetDirectoryName(targetPath);
Directory.CreateDirectory(directory);
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken))
{
tracer.Info($"Downloading: {targetPath}");
using (var targetStream = new FileStream(targetPath, FileMode.Create))
{
await sourceStream.CopyToAsync(targetStream);
}
}
},
maxRetries: 5,
cancellationToken: cancellationToken,
tracer: tracer,
continueOnCapturedContext: false,
canRetryDelegate: exception => exception is IOException,
context: null
);
},
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 5000,
MaxDegreeOfParallelism = 8,
CancellationToken = cancellationToken,
});
await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken);
}
private async Task DownloadFileContainerAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true)
{
var containerIdAndRoot = ParseContainerId(artifact.Resource.Data);
var projectId = downloadParameters.ProjectId;
var minimatchPatterns = downloadParameters.MinimatchFilters;
var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, isShallow : false, includeBlobMetadata : true, containerIdAndRoot.Item2);
tracer.Info($"Start downloading FCS artifact- {artifact.Name}");
IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer);
if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0)
{
items = this.GetFilteredItems(items, minimatcherFuncs);
}
if (!isSingleArtifactDownload && items.Any())
{
Directory.CreateDirectory(rootPath);
}
var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder);
Parallel.ForEach(folderItems, (folder) =>
{
var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
Directory.CreateDirectory(targetPath);
});
var fileItems = items.Where(i => i.ItemType == ContainerItemType.File);
var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>(
async item =>
{
var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
var directory = Path.GetDirectoryName(targetPath);
Directory.CreateDirectory(directory);
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
tracer.Info($"Downloading: {targetPath}");
if (item.BlobMetadata != null)
{
await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, cancellationToken);
}
else
{
using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken))
using (var targetStream = new FileStream(targetPath, FileMode.Create))
{
await sourceStream.CopyToAsync(targetStream);
}
}
},
maxRetries: downloadParameters.RetryDownloadCount,
cancellationToken: cancellationToken,
tracer: tracer,
continueOnCapturedContext: false,
canRetryDelegate: exception => exception is IOException,
context: null
);
},
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 5000,
MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit,
CancellationToken = cancellationToken,
});
await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken);
// check files (will throw an exception if a file is corrupt)
if (downloadParameters.CheckDownloadedFiles)
{
CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
}
}
// Download with minimatch patterns, V1.
internal async Task DownloadAsync(
AgentTaskPluginExecutionContext context,
PipelineArtifactDownloadParameters downloadParameters,
DownloadOptions downloadOptions,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
BlobStoreClientTelemetry clientTelemetry;
DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry);
BuildServer buildServer = new BuildServer(connection);
using (clientTelemetry)
{
// download all pipeline artifacts if artifact name is missing
if (downloadOptions == DownloadOptions.MultiDownload)
{
List <BuildArtifact> artifacts;
if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId)
{
artifacts = await buildServer.GetArtifactsAsync(downloadParameters.ProjectId, downloadParameters.PipelineId, cancellationToken);
}
else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName)
{
if (string.IsNullOrEmpty(downloadParameters.ProjectName))
{
throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!");
}
else
{
artifacts = await buildServer.GetArtifactsWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, cancellationToken);
}
}
else
{
throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!");
}
IEnumerable <BuildArtifact> pipelineArtifacts = artifacts.Where(a => string.Equals(a.Resource.Type, PipelineArtifactConstants.PipelineArtifact, StringComparison.OrdinalIgnoreCase));
if (pipelineArtifacts.Count() == 0)
{
throw new ArgumentException("Could not find any pipeline artifacts in the build.");
}
else
{
context.Output(StringUtil.Loc("DownloadingMultiplePipelineArtifacts", pipelineArtifacts.Count()));
var artifactNameAndManifestIds = pipelineArtifacts.ToDictionary(
keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong
elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data));
// 2) download to the target path
var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds(
artifactNameAndManifestIds,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters,
minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName);
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true,
context: nameof(DownloadAsync),
cancellationToken: cancellationToken,
continueOnCapturedContext: false);
});
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
}
else if (downloadOptions == DownloadOptions.SingleDownload)
{
// 1) get manifest id from artifact data
BuildArtifact buildArtifact;
if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectId)
{
buildArtifact = await buildServer.GetArtifact(downloadParameters.ProjectId, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken);
}
else if (downloadParameters.ProjectRetrievalOptions == BuildArtifactRetrievalOptions.RetrieveByProjectName)
{
if (string.IsNullOrEmpty(downloadParameters.ProjectName))
{
throw new InvalidOperationException("Project name can't be empty when trying to fetch build artifacts!");
}
else
{
buildArtifact = await buildServer.GetArtifactWithProjectNameAsync(downloadParameters.ProjectName, downloadParameters.PipelineId, downloadParameters.ArtifactName, cancellationToken);
}
}
else
{
throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!");
}
var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data);
var options = DownloadDedupManifestArtifactOptions.CreateWithManifestId(
manifestId,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters);
PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true,
context: nameof(DownloadAsync),
cancellationToken: cancellationToken,
continueOnCapturedContext: false);
});
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
else
{
throw new InvalidOperationException($"Invalid {nameof(downloadOptions)}!");
}
}
}
/// <summary>
/// Alternative for UploadAndReferenceBlobAsync that uses WalkBlocksAsync instead of the synchronous WalkBlocks.
/// Also utilizes the AsyncHttpRetryHelper to mitigate transient exceptions.
/// </summary>
public static Task UploadAndReferenceBlobWithRetriesAsync(
this IBlobStoreHttpClient client,
BlobIdentifier blobId,
Stream stream,
BlobReference reference,
Context context,
CancellationToken cts)
{
Contract.Requires(stream != null);
var attempt = 0;
return(AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
bool blobUploaded = false;
stream.Position = 0;
attempt++;
await VsoHash.WalkBlocksAsync(
stream,
blockActionSemaphore: null,
multiBlocksInParallel: true,
singleBlockCallback:
async(block, blockLength, blockHash) =>
{
await client.PutSingleBlockBlobAndReferenceAsync(
blobId, block, blockLength, reference, cts).ConfigureAwait(false);
blobUploaded = true;
},
multiBlockCallback:
(block, blockLength, blockHash, isFinalBlock) =>
client.PutBlobBlockAsync(blobId, block, blockLength, cts),
multiBlockSealCallback: async blobIdWithBlocks =>
{
var failedRefs = await client.TryReferenceWithBlocksAsync(
new Dictionary <BlobIdentifierWithBlocks, IEnumerable <BlobReference> >
{
{ blobIdWithBlocks, new[] { reference } }
},
cancellationToken: cts).ConfigureAwait(false);
blobUploaded = !failedRefs.Any();
}
);
if (!blobUploaded)
{
throw new AsyncHttpRetryHelper.RetryableException($"Could not upload blob on attempt {attempt}.");
}
},
maxRetries: 5,
tracer: new AppTraceSourceContextAdapter(context, "BlobStoreHttpClient", SourceLevels.All),
canRetryDelegate: exception =>
{
// HACK HACK: This is an additional layer of retries specifically to catch the SSL exceptions seen in DM.
// Newer versions of Artifact packages have this retry automatically, but the packages deployed with the M119.0 release do not.
// Once the next release is completed, these retries can be removed.
if (exception is HttpRequestException && exception.InnerException is WebException)
{
return true;
}
while (exception != null)
{
if (exception is TimeoutException)
{
return true;
}
exception = exception.InnerException;
}
return false;
},
cancellationToken: cts,
continueOnCapturedContext: false,
context: context.Id.ToString()));
}
private async Task DownloadFileContainerAsync(IEnumerable <FileContainerItem> items, ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true)
{
var containerIdAndRoot = ParseContainerId(artifact.Resource.Data);
var projectId = downloadParameters.ProjectId;
tracer.Info($"Start downloading FCS artifact- {artifact.Name}");
if (!isSingleArtifactDownload && items.Any())
{
Directory.CreateDirectory(rootPath);
}
var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder);
Parallel.ForEach(folderItems, (folder) =>
{
var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
Directory.CreateDirectory(targetPath);
});
var fileItems = items.Where(i => i.ItemType == ContainerItemType.File);
// Only initialize these clients if we know we need to download from Blobstore
// If a client cannot connect to Blobstore, we shouldn't stop them from downloading from FCS
var downloadFromBlob = !AgentKnobs.DisableBuildArtifactsToBlob.GetValue(context).AsBoolean();
DedupStoreClient dedupClient = null;
BlobStoreClientTelemetryTfs clientTelemetry = null;
if (downloadFromBlob && fileItems.Any(x => x.BlobMetadata != null))
{
try
{
(dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupClientAsync(
false, (str) => this.tracer.Info(str), this.connection, DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context), cancellationToken);
}
catch (SocketException e)
{
ExceptionsUtil.HandleSocketException(e, connection.Uri.ToString(), context.Warning);
}
catch
{
var blobStoreHost = dedupClient.Client.BaseAddress.Host;
var allowListLink = BlobStoreWarningInfoProvider.GetAllowListLinkForCurrentPlatform();
var warningMessage = StringUtil.Loc("BlobStoreDownloadWarning", blobStoreHost, allowListLink);
// Fall back to streaming through TFS if we cannot reach blobstore
downloadFromBlob = false;
tracer.Warn(warningMessage);
}
}
var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>(
async item =>
{
var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
var directory = Path.GetDirectoryName(targetPath);
Directory.CreateDirectory(directory);
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
tracer.Info($"Downloading: {targetPath}");
if (item.BlobMetadata != null && downloadFromBlob)
{
await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, dedupClient, clientTelemetry, cancellationToken);
}
else
{
using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken))
using (var targetStream = new FileStream(targetPath, FileMode.Create))
{
await sourceStream.CopyToAsync(targetStream);
}
}
},
maxRetries: downloadParameters.RetryDownloadCount,
cancellationToken: cancellationToken,
tracer: tracer,
continueOnCapturedContext: false,
canRetryDelegate: exception => exception is IOException,
context: null
);
},
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 5000,
MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit,
CancellationToken = cancellationToken,
});
await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken);
// Send results to CustomerIntelligence
if (clientTelemetry != null)
{
var planId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.PlanId)?.Value ?? Guid.Empty.ToString());
var jobId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.JobId)?.Value ?? Guid.Empty.ToString());
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.BuildArtifactDownload,
properties: clientTelemetry.GetArtifactDownloadTelemetry(planId, jobId));
}
// check files (will throw an exception if a file is corrupt)
if (downloadParameters.CheckDownloadedFiles)
{
CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
}
}
public static async Task <(List <BlobFileInfo> fileDedupIds, ulong length)> UploadBatchToBlobstore(
bool verbose,
IReadOnlyList <string> itemPaths,
Func <TelemetryInformationLevel, Uri, string, BlobStoreTelemetryRecord> telemetryRecordFactory,
Action <string> traceOutput,
DedupStoreClient dedupClient,
BlobStoreClientTelemetry clientTelemetry,
CancellationToken cancellationToken,
bool enableReporting = false)
{
// Create chunks and identifier
traceOutput(StringUtil.Loc("BuildingFileTree"));
var fileNodes = await GenerateHashes(itemPaths, cancellationToken);
var rootNode = CreateNodeToUpload(fileNodes.Where(x => x.Success).Select(y => y.Node));
// If there are multiple paths to one DedupId (duplicate files)
// take the last one
var fileDedupIds = new Dictionary <DedupIdentifier, string>();
foreach (var file in fileNodes.Where(x => x.Success))
{
// ChunkHelper uses 64k block default size
var dedupId = file.Node.GetDedupIdentifier(HashType.Dedup64K);
fileDedupIds[dedupId] = file.Path;
}
// Setup upload session to keep file for at mimimum one day
// Blobs will need to be associated with the server with an ID ref otherwise they will be
// garbage collected after one day
var tracer = DedupManifestArtifactClientFactory.CreateArtifactsTracer(verbose, traceOutput);
var keepUntilRef = new KeepUntilBlobReference(DateTime.UtcNow.AddDays(1));
var uploadSession = dedupClient.CreateUploadSession(keepUntilRef, tracer, FileSystem.Instance);
using (var reportingCancelSrc = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
// Log stats
Task reportingTask = null;
if (enableReporting)
{
reportingTask = StartReportingTask(traceOutput, (long)rootNode.TransitiveContentBytes, uploadSession, reportingCancelSrc);
}
// Upload the chunks
var uploadRecord = clientTelemetry.CreateRecord <BlobStoreTelemetryRecord>(telemetryRecordFactory);
await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() => await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
await uploadSession.UploadAsync(rootNode, fileDedupIds, cancellationToken);
return(uploadSession.UploadStatistics);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
cancellationToken: cancellationToken,
continueOnCapturedContext: false)
);
if (enableReporting)
{
reportingCancelSrc.Cancel();
await reportingTask;
}
}
return(fileNodes, rootNode.TransitiveContentBytes);
}
private async Task <int> GetPipelineIdAsync(
AgentTaskPluginExecutionContext context,
string pipelineDefinition,
string pipelineVersionToDownload,
string project,
string[] tagFilters,
BuildResult resultFilter = BuildResult.Succeeded,
string branchName = null,
CancellationToken cancellationToken = default(CancellationToken))
{
if (String.IsNullOrWhiteSpace(pipelineDefinition))
{
throw new InvalidOperationException(StringUtil.Loc("CannotBeNullOrEmpty", "Pipeline Definition"));
}
VssConnection connection = context.VssConnection;
BuildHttpClient buildHttpClient = connection.GetClient <BuildHttpClient>();
var isDefinitionNum = Int32.TryParse(pipelineDefinition, out int definition);
if (!isDefinitionNum)
{
var definitionReferencesWithName = await AsyncHttpRetryHelper.InvokeAsync(
async() => await buildHttpClient.GetDefinitionsAsync(new Guid(project), pipelineDefinition, cancellationToken: cancellationToken),
maxRetries : MaxRetries,
tracer : tracer,
context : "GetBuildDefinitionReferencesByName",
cancellationToken : cancellationToken,
continueOnCapturedContext : false);
var definitionRef = definitionReferencesWithName.FirstOrDefault();
if (definitionRef == null)
{
throw new ArgumentException(StringUtil.Loc("PipelineDoesNotExist", pipelineDefinition));
}
else
{
definition = definitionRef.Id;
}
}
var definitions = new List <int>()
{
definition
};
List <Build> list;
if (pipelineVersionToDownload == pipelineVersionToDownloadLatest)
{
list = await AsyncHttpRetryHelper.InvokeAsync(
async() => await buildHttpClient.GetBuildsAsync(
project,
definitions,
tagFilters: tagFilters,
queryOrder: BuildQueryOrder.FinishTimeDescending,
resultFilter: resultFilter,
cancellationToken: cancellationToken),
maxRetries : MaxRetries,
tracer : tracer,
context : "GetLatestBuild",
cancellationToken : cancellationToken,
continueOnCapturedContext : false);
}
else if (pipelineVersionToDownload == pipelineVersionToDownloadLatestFromBranch)
{
list = await AsyncHttpRetryHelper.InvokeAsync(
async() => await buildHttpClient.GetBuildsAsync(
project,
definitions,
branchName: branchName,
tagFilters: tagFilters,
queryOrder: BuildQueryOrder.FinishTimeDescending,
resultFilter: resultFilter,
cancellationToken: cancellationToken),
maxRetries : MaxRetries,
tracer : tracer,
context : "GetLatestBuildFromBranch",
cancellationToken : cancellationToken,
continueOnCapturedContext : false);
}
else
{
throw new InvalidOperationException("Unreachable code!");
}
if (list.Count > 0)
{
return(list.First().Id);
}
else
{
throw new ArgumentException(StringUtil.Loc("BuildsDoesNotExist"));
}
}
internal async Task UploadAsync(
AgentTaskPluginExecutionContext context,
Fingerprint fingerprint,
string path,
CancellationToken cancellationToken,
ContentFormat contentFormat)
{
VssConnection connection = context.VssConnection;
var(dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance
.CreateDedupManifestClientAsync(context.IsSystemDebugTrue(), (str) => context.Output(str), connection, cancellationToken);
PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection);
using (clientTelemetry)
{
// Check if the key exists.
PipelineCacheActionRecord cacheRecordGet = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context));
PipelineCacheArtifact getResult = await pipelineCacheClient.GetPipelineCacheArtifactAsync(new [] { fingerprint }, cancellationToken, cacheRecordGet);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecordGet);
//If cache exists, return.
if (getResult != null)
{
context.Output($"Cache with fingerprint `{getResult.Fingerprint}` already exists.");
return;
}
string uploadPath = await this.GetUploadPathAsync(contentFormat, context, path, cancellationToken);
//Upload the pipeline artifact.
PipelineCacheActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, nameof(dedupManifestClient.PublishAsync), context));
PublishResult result = await clientTelemetry.MeasureActionAsync(
record : uploadRecord,
actionAsync : async() =>
await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
return(await dedupManifestClient.PublishAsync(uploadPath, cancellationToken));
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
cancellationToken: cancellationToken,
continueOnCapturedContext: false)
);
CreatePipelineCacheArtifactContract options = new CreatePipelineCacheArtifactContract
{
Fingerprint = fingerprint,
RootId = result.RootId,
ManifestId = result.ManifestId,
ProofNodes = result.ProofNodes.ToArray(),
ContentFormat = contentFormat.ToString(),
};
// delete archive file if it's tar.
if (contentFormat == ContentFormat.SingleTar)
{
try
{
if (File.Exists(uploadPath))
{
File.Delete(uploadPath);
}
}
catch { }
}
// Cache the artifact
PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.SaveCache, context));
CreateStatus status = await pipelineCacheClient.CreatePipelineCacheArtifactAsync(options, cancellationToken, cacheRecord);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: uploadRecord);
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord);
context.Output("Saved item.");
}
}
private async Task DownloadFileContainerAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true)
{
var containerIdAndRoot = ParseContainerId(artifact.Resource.Data);
var projectId = downloadParameters.ProjectId;
var minimatchPatterns = downloadParameters.MinimatchFilters;
var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, isShallow : false, includeBlobMetadata : true, containerIdAndRoot.Item2);
tracer.Info($"Start downloading FCS artifact- {artifact.Name}");
IEnumerable <Func <string, bool> > minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer, downloadParameters.CustomMinimatchOptions);
if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0)
{
items = this.GetFilteredItems(items, minimatcherFuncs);
}
if (!isSingleArtifactDownload && items.Any())
{
Directory.CreateDirectory(rootPath);
}
var folderItems = items.Where(i => i.ItemType == ContainerItemType.Folder);
Parallel.ForEach(folderItems, (folder) =>
{
var targetPath = ResolveTargetPath(rootPath, folder, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
Directory.CreateDirectory(targetPath);
});
var fileItems = items.Where(i => i.ItemType == ContainerItemType.File);
// Only initialize these clients if we know we need to download from Blobstore
// If a client cannot connect to Blobstore, we shouldn't stop them from downloading from FCS
var downloadFromBlob = !AgentKnobs.DisableBuildArtifactsToBlob.GetValue(context).AsBoolean();
DedupStoreClient dedupClient = null;
BlobStoreClientTelemetryTfs clientTelemetry = null;
if (downloadFromBlob && fileItems.Any(x => x.BlobMetadata != null))
{
(dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupClientAsync(
false, (str) => this.tracer.Info(str), this.connection, cancellationToken);
}
var downloadBlock = NonSwallowingActionBlock.Create <FileContainerItem>(
async item =>
{
var targetPath = ResolveTargetPath(rootPath, item, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
var directory = Path.GetDirectoryName(targetPath);
Directory.CreateDirectory(directory);
await AsyncHttpRetryHelper.InvokeVoidAsync(
async() =>
{
tracer.Info($"Downloading: {targetPath}");
if (item.BlobMetadata != null && downloadFromBlob)
{
await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, dedupClient, clientTelemetry, cancellationToken);
}
else
{
using (var sourceStream = await this.DownloadFileAsync(containerIdAndRoot, projectId, containerClient, item, cancellationToken))
using (var targetStream = new FileStream(targetPath, FileMode.Create))
{
await sourceStream.CopyToAsync(targetStream);
}
}
},
maxRetries: downloadParameters.RetryDownloadCount,
cancellationToken: cancellationToken,
tracer: tracer,
continueOnCapturedContext: false,
canRetryDelegate: exception => exception is IOException,
context: null
);
},
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 5000,
MaxDegreeOfParallelism = downloadParameters.ParallelizationLimit,
CancellationToken = cancellationToken,
});
await downloadBlock.SendAllAndCompleteSingleBlockNetworkAsync(fileItems, cancellationToken);
// Send results to CustomerIntelligence
if (clientTelemetry != null)
{
var planId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.PlanId)?.Value ?? Guid.Empty.ToString());
var jobId = new Guid(context.Variables.GetValueOrDefault(WellKnownDistributedTaskVariables.JobId)?.Value ?? Guid.Empty.ToString());
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.BuildArtifactDownload,
properties: clientTelemetry.GetArtifactDownloadTelemetry(planId, jobId));
}
// check files (will throw an exception if a file is corrupt)
if (downloadParameters.CheckDownloadedFiles)
{
CheckDownloads(items, rootPath, containerIdAndRoot.Item2, downloadParameters.IncludeArtifactNameInPath);
}
}
internal async Task DownloadAsync(
AgentTaskPluginExecutionContext context,
Fingerprint[] fingerprints,
string path,
string cacheHitVariable,
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
var(dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance
.CreateDedupManifestClientAsync(context.IsSystemDebugTrue(), (str) => context.Output(str), connection, cancellationToken);
PipelineCacheClient pipelineCacheClient = await this.CreateClientWithRetryAsync(clientTelemetry, context, connection, cancellationToken);
using (clientTelemetry)
{
PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context));
PipelineCacheArtifact result = await AsyncHttpRetryHelper.InvokeAsync(
async() =>
{
return(await pipelineCacheClient.GetPipelineCacheArtifactAsync(fingerprints, cancellationToken, cacheRecord));
},
maxRetries : 3,
tracer : tracer,
canRetryDelegate : e => true, // this isn't great, but failing on upload stinks, so just try a couple of times
cancellationToken : cancellationToken,
continueOnCapturedContext : false);
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord);
if (result != null)
{
context.Output($"Entry found at fingerprint: `{result.Fingerprint.ToString()}`");
context.Verbose($"Manifest ID is: {result.ManifestId.ValueString}");
PipelineCacheActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) =>
new PipelineCacheActionRecord(level, uri, type, nameof(DownloadAsync), context));
await clientTelemetry.MeasureActionAsync(
record : downloadRecord,
actionAsync : async() =>
{
await this.DownloadPipelineCacheAsync(context, dedupManifestClient, result.ManifestId, path, Enum.Parse <ContentFormat>(result.ContentFormat), cancellationToken);
});
// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: downloadRecord);
context.Output("Cache restored.");
}
if (!string.IsNullOrEmpty(cacheHitVariable))
{
if (result == null)
{
context.SetVariable(cacheHitVariable, "false");
}
else
{
context.Verbose($"Exact fingerprint: `{result.Fingerprint.ToString()}`");
bool foundExact = false;
foreach (var fingerprint in fingerprints)
{
context.Verbose($"This fingerprint: `{fingerprint.ToString()}`");
if (fingerprint == result.Fingerprint ||
result.Fingerprint.Segments.Length == 1 && result.Fingerprint.Segments.Single() == fingerprint.SummarizeForV1())
{
foundExact = true;
break;
}
}
context.SetVariable(cacheHitVariable, foundExact ? "true" : "inexact");
}
}
}
}
