internal async Task UploadAsync( AgentTaskPluginExecutionContext context, Fingerprint fingerprint, string path, CancellationToken cancellationToken, ContentFormat contentFormat) { VssConnection connection = context.VssConnection; var(dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance .CreateDedupManifestClientAsync(context.IsSystemDebugTrue(), (str) => context.Output(str), connection, DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context), cancellationToken); PipelineCacheClient pipelineCacheClient = await this.CreateClientWithRetryAsync(clientTelemetry, context, connection, cancellationToken); using (clientTelemetry) { // Check if the key exists. PipelineCacheActionRecord cacheRecordGet = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context)); PipelineCacheArtifact getResult = await pipelineCacheClient.GetPipelineCacheArtifactAsync(new [] { fingerprint }, cancellationToken, cacheRecordGet); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecordGet); //If cache exists, return. if (getResult != null) { context.Output($"Cache with fingerprint `{getResult.Fingerprint}` already exists."); return; } string uploadPath = await this.GetUploadPathAsync(contentFormat, context, path, cancellationToken); //Upload the pipeline artifact. PipelineCacheActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, nameof(dedupManifestClient.PublishAsync), context)); PublishResult result = await clientTelemetry.MeasureActionAsync( record : uploadRecord, actionAsync : async() => await AsyncHttpRetryHelper.InvokeAsync( async() => { return(await dedupManifestClient.PublishAsync(uploadPath, cancellationToken)); }, maxRetries: 3, tracer: tracer, canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times cancellationToken: cancellationToken, continueOnCapturedContext: false) ); CreatePipelineCacheArtifactContract options = new CreatePipelineCacheArtifactContract { Fingerprint = fingerprint, RootId = result.RootId, ManifestId = result.ManifestId, ProofNodes = result.ProofNodes.ToArray(), ContentFormat = contentFormat.ToString(), }; // delete archive file if it's tar. if (contentFormat == ContentFormat.SingleTar) { try { if (File.Exists(uploadPath)) { File.Delete(uploadPath); } } catch { } } // Cache the artifact PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.SaveCache, context)); CreateStatus status = await pipelineCacheClient.CreatePipelineCacheArtifactAsync(options, cancellationToken, cacheRecord); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: uploadRecord); context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord); context.Output("Saved item."); } }
internal async Task DownloadAsync( AgentTaskPluginExecutionContext context, Fingerprint[] fingerprints, string path, string cacheHitVariable, CancellationToken cancellationToken) { VssConnection connection = context.VssConnection; var(dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance .CreateDedupManifestClientAsync(context.IsSystemDebugTrue(), (str) => context.Output(str), connection, DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context), cancellationToken); PipelineCacheClient pipelineCacheClient = await this.CreateClientWithRetryAsync(clientTelemetry, context, connection, cancellationToken); using (clientTelemetry) { PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context)); PipelineCacheArtifact result = await pipelineCacheClient.GetPipelineCacheArtifactAsync(fingerprints, cancellationToken, cacheRecord); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord); if (result != null) { context.Output($"Entry found at fingerprint: `{result.Fingerprint.ToString()}`"); context.Verbose($"Manifest ID is: {result.ManifestId.ValueString}"); PipelineCacheActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, nameof(DownloadAsync), context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await this.DownloadPipelineCacheAsync(context, dedupManifestClient, result.ManifestId, path, Enum.Parse <ContentFormat>(result.ContentFormat), cancellationToken); }); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: downloadRecord); context.Output("Cache restored."); } if (!string.IsNullOrEmpty(cacheHitVariable)) { if (result == null) { context.SetVariable(cacheHitVariable, "false"); } else { context.Verbose($"Exact fingerprint: `{result.Fingerprint.ToString()}`"); bool foundExact = false; foreach (var fingerprint in fingerprints) { context.Verbose($"This fingerprint: `{fingerprint.ToString()}`"); if (fingerprint == result.Fingerprint || result.Fingerprint.Segments.Length == 1 && result.Fingerprint.Segments.Single() == fingerprint.SummarizeForV1()) { foundExact = true; break; } } context.SetVariable(cacheHitVariable, foundExact ? "true" : "inexact"); } } } }
internal async Task UploadAsync( AgentTaskPluginExecutionContext context, IEnumerable <string> key, string path, string salt, CancellationToken cancellationToken) { VssConnection connection = context.VssConnection; BlobStoreClientTelemetry clientTelemetry; DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, out clientTelemetry); PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection); using (clientTelemetry) { // Check if the key exists. GetPipelineCacheArtifactOptions getOptions = new GetPipelineCacheArtifactOptions { Key = key, Salt = salt, }; PipelineCacheActionRecord cacheRecordGet = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context)); PipelineCacheArtifact getResult = await pipelineCacheClient.GetPipelineCacheArtifactAsync(getOptions, cancellationToken, cacheRecordGet); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecordGet); //If cache exists, return. if (getResult != null) { context.Output($"Cache with fingerprint {getResult.Fingerprint} already exists."); return; } //Upload the pipeline artifact. PipelineCacheActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, nameof(dedupManifestClient.PublishAsync), context)); PublishResult result = await clientTelemetry.MeasureActionAsync( record : uploadRecord, actionAsync : async() => { return(await dedupManifestClient.PublishAsync(path, cancellationToken)); }); CreatePipelineCacheArtifactOptions options = new CreatePipelineCacheArtifactOptions { Key = key, RootId = result.RootId, ManifestId = result.ManifestId, ProofNodes = result.ProofNodes.ToArray(), Salt = salt }; // Cache the artifact PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.SaveCache, context)); CreateStatus status = await pipelineCacheClient.CreatePipelineCacheArtifactAsync(options, cancellationToken, cacheRecord); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: uploadRecord); context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord); context.Output("Saved item."); } }
internal async Task UploadAsync( AgentTaskPluginExecutionContext context, Fingerprint fingerprint, string path, CancellationToken cancellationToken, ContentFormat contentFormat) { VssConnection connection = context.VssConnection; BlobStoreClientTelemetry clientTelemetry; DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry); PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection); using (clientTelemetry) { // Check if the key exists. PipelineCacheActionRecord cacheRecordGet = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context)); PipelineCacheArtifact getResult = await pipelineCacheClient.GetPipelineCacheArtifactAsync(new [] { fingerprint }, cancellationToken, cacheRecordGet); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecordGet); //If cache exists, return. if (getResult != null) { context.Output($"Cache with fingerprint `{getResult.Fingerprint}` already exists."); return; } string uploadPath = await this.GetUploadPathAsync(contentFormat, context, path, cancellationToken); //Upload the pipeline artifact. PipelineCacheActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, nameof(dedupManifestClient.PublishAsync), context)); PublishResult result = await clientTelemetry.MeasureActionAsync( record : uploadRecord, actionAsync : async() => { return(await dedupManifestClient.PublishAsync(uploadPath, cancellationToken)); }); CreatePipelineCacheArtifactContract options = new CreatePipelineCacheArtifactContract { Fingerprint = fingerprint, RootId = result.RootId, ManifestId = result.ManifestId, ProofNodes = result.ProofNodes.ToArray(), ContentFormat = contentFormat.ToString(), }; // delete archive file if it's tar. if (contentFormat == ContentFormat.SingleTar) { try { if (File.Exists(uploadPath)) { File.Delete(uploadPath); } } catch { } } // Cache the artifact PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.SaveCache, context)); CreateStatus status = await pipelineCacheClient.CreatePipelineCacheArtifactAsync(options, cancellationToken, cacheRecord); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: uploadRecord); context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord); context.Output("Saved item."); } }
internal async Task DownloadAsync( AgentTaskPluginExecutionContext context, Fingerprint[] fingerprints, string path, string cacheHitVariable, CancellationToken cancellationToken) { VssConnection connection = context.VssConnection; BlobStoreClientTelemetry clientTelemetry; DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry); PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection); using (clientTelemetry) { PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context)); PipelineCacheArtifact result = await pipelineCacheClient.GetPipelineCacheArtifactAsync(fingerprints, cancellationToken, cacheRecord); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord); if (result != null) { context.Output($"Entry found at fingerprint: `{result.Fingerprint.ToString()}`"); context.Verbose($"Manifest ID is: {result.ManifestId.ValueString}"); PipelineCacheActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, nameof(DownloadAsync), context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await this.DownloadPipelineCacheAsync(dedupManifestClient, result.ManifestId, path, cancellationToken); }); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: downloadRecord); context.Output("Cache restored."); } if (!string.IsNullOrEmpty(cacheHitVariable)) { if (result == null) { context.SetVariable(cacheHitVariable, "false"); } else { context.Verbose($"Exact fingerprint: `{result.Fingerprint.ToString()}`"); bool foundExact = false; foreach (var fingerprint in fingerprints) { context.Verbose($"This fingerprint: `{fingerprint.ToString()}`"); if (fingerprint == result.Fingerprint) { foundExact = true; break; } } context.SetVariable(cacheHitVariable, foundExact ? "true" : "inexact"); } } } }