internal async Task DownloadAsync( AgentTaskPluginExecutionContext context, IEnumerable <string> key, string path, string salt, string variableToSetOnHit, CancellationToken cancellationToken) { VssConnection connection = context.VssConnection; var pipelineCacheClient = this.CreateClient(context, connection); GetPipelineCacheArtifactOptions options = new GetPipelineCacheArtifactOptions { Key = key, Scope = "myscope", Salt = salt, }; var result = await pipelineCacheClient.GetPipelineCacheArtifactAsync(options, cancellationToken); if (result == null) { return; } else { Console.WriteLine("Manifest ID is: {0}", result.ManifestId.ValueString); DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection); await this.DownloadPipelineCacheAsync(dedupManifestClient, result.ManifestId, path, cancellationToken); context.SetVariable($"{PipelineCacheVarPrefix}.{variableToSetOnHit}", "True"); Console.WriteLine("Cache restored."); } }
internal async Task UploadAsync( AgentTaskPluginExecutionContext context, IEnumerable <string> key, string path, string salt, CancellationToken cancellationToken) { VssConnection connection = context.VssConnection; DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection); var result = await dedupManifestClient.PublishAsync(path, cancellationToken); var scope = "myscope"; CreatePipelineCacheArtifactOptions options = new CreatePipelineCacheArtifactOptions { Key = key, RootId = result.RootId, ManifestId = result.ManifestId, Scope = scope, ProofNodes = result.ProofNodes.ToArray(), Salt = salt }; var pipelineCacheClient = this.CreateClient(context, connection); await pipelineCacheClient.CreatePipelineCacheArtifactAsync(options, cancellationToken); Console.WriteLine("Saved item."); }
internal async Task UploadAsync( AgentTaskPluginExecutionContext context, Fingerprint fingerprint, string path, CancellationToken cancellationToken) { VssConnection connection = context.VssConnection; BlobStoreClientTelemetry clientTelemetry; DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry); PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection); using (clientTelemetry) { // Check if the key exists. PipelineCacheActionRecord cacheRecordGet = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context)); PipelineCacheArtifact getResult = await pipelineCacheClient.GetPipelineCacheArtifactAsync(new [] { fingerprint }, cancellationToken, cacheRecordGet); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecordGet); //If cache exists, return. if (getResult != null) { context.Output($"Cache with fingerprint `{getResult.Fingerprint}` already exists."); return; } //Upload the pipeline artifact. PipelineCacheActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, nameof(dedupManifestClient.PublishAsync), context)); PublishResult result = await clientTelemetry.MeasureActionAsync( record : uploadRecord, actionAsync : async() => { return(await dedupManifestClient.PublishAsync(path, cancellationToken)); }); CreatePipelineCacheArtifactOptions options = new CreatePipelineCacheArtifactOptions { Fingerprint = fingerprint, RootId = result.RootId, ManifestId = result.ManifestId, ProofNodes = result.ProofNodes.ToArray(), }; // Cache the artifact PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.SaveCache, context)); CreateStatus status = await pipelineCacheClient.CreatePipelineCacheArtifactAsync(options, cancellationToken, cacheRecord); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: uploadRecord); context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord); context.Output("Saved item."); } }
internal async Task DownloadAsync( AgentTaskPluginExecutionContext context, IEnumerable <string> key, string path, string salt, string variableToSetOnHit, CancellationToken cancellationToken) { VssConnection connection = context.VssConnection; BlobStoreClientTelemetry clientTelemetry; DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, out clientTelemetry); PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection); GetPipelineCacheArtifactOptions options = new GetPipelineCacheArtifactOptions { Key = key, Salt = salt, }; using (clientTelemetry) { PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context)); PipelineCacheArtifact result = await pipelineCacheClient.GetPipelineCacheArtifactAsync(options, cancellationToken, cacheRecord); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord); if (result == null) { return; } else { context.Output($"Manifest ID is: {result.ManifestId.ValueString}"); PipelineCacheActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, nameof(DownloadAsync), context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await this.DownloadPipelineCacheAsync(dedupManifestClient, result.ManifestId, path, cancellationToken); }); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: downloadRecord); if (!string.IsNullOrEmpty(variableToSetOnHit)) { context.SetVariable($"{PipelineArtifactConstants.PipelineCache}.{variableToSetOnHit}", "True"); } Console.WriteLine("Cache restored."); } } }
public static async Task <(DedupIdentifier dedupId, ulong length)> UploadToBlobStore( bool verbose, string itemPath, Func <TelemetryInformationLevel, Uri, string, BlobStoreTelemetryRecord> telemetryRecordFactory, Action <string> traceOutput, DedupStoreClient dedupClient, BlobStoreClientTelemetry clientTelemetry, CancellationToken cancellationToken) { // Create chunks and identifier var chunk = await ChunkerHelper.CreateFromFileAsync(FileSystem.Instance, itemPath, cancellationToken, false); var rootNode = new DedupNode(new [] { chunk }); // ChunkHelper uses 64k block default size var dedupId = rootNode.GetDedupIdentifier(HashType.Dedup64K); // Setup upload session to keep file for at mimimum one day // Blobs will need to be associated with the server with an ID ref otherwise they will be // garbage collected after one day var tracer = DedupManifestArtifactClientFactory.CreateArtifactsTracer(verbose, traceOutput); var keepUntilRef = new KeepUntilBlobReference(DateTime.UtcNow.AddDays(1)); var uploadSession = dedupClient.CreateUploadSession(keepUntilRef, tracer, FileSystem.Instance); // Upload the chunks var uploadRecord = clientTelemetry.CreateRecord <BlobStoreTelemetryRecord>(telemetryRecordFactory); await clientTelemetry.MeasureActionAsync( record : uploadRecord, actionAsync : async() => await AsyncHttpRetryHelper.InvokeAsync( async() => { await uploadSession.UploadAsync(rootNode, new Dictionary <DedupIdentifier, string>() { [dedupId] = itemPath }, cancellationToken); return(uploadSession.UploadStatistics); }, maxRetries: 3, tracer: tracer, canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times cancellationToken: cancellationToken, continueOnCapturedContext: false) ); return(dedupId, rootNode.TransitiveContentBytes); }
internal async Task UploadAsync( AgentTaskPluginExecutionContext context, IEnumerable <string> key, string path, string salt, CancellationToken cancellationToken) { VssConnection connection = context.VssConnection; BlobStoreClientTelemetry clientTelemetry; DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, out clientTelemetry); PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection); using (clientTelemetry) { //Upload the pipeline artifact. PipelineCacheActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, nameof(dedupManifestClient.PublishAsync), context)); PublishResult result = await clientTelemetry.MeasureActionAsync( record : uploadRecord, actionAsync : async() => { return(await dedupManifestClient.PublishAsync(path, cancellationToken)); }); CreatePipelineCacheArtifactOptions options = new CreatePipelineCacheArtifactOptions { Key = key, RootId = result.RootId, ManifestId = result.ManifestId, ProofNodes = result.ProofNodes.ToArray(), Salt = salt }; // Cache the artifact PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.SaveCache, context)); CreateStatus status = await pipelineCacheClient.CreatePipelineCacheArtifactAsync(options, cancellationToken, cacheRecord); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: uploadRecord); context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord); context.Output("Saved item."); } }
private async Task <(DedupIdentifier dedupId, ulong length)> UploadToBlobStore(IAsyncCommandContext context, string itemPath, CancellationToken cancellationToken) { // Create chunks and identifier var chunk = await ChunkerHelper.CreateFromFileAsync(FileSystem.Instance, itemPath, cancellationToken, false); var rootNode = new DedupNode(new [] { chunk }); var dedupId = rootNode.GetDedupIdentifier(HashType.Dedup64K); // Setup upload session to keep file for at mimimum one day var verbose = String.Equals(context.GetVariableValueOrDefault("system.debug"), "true", StringComparison.InvariantCultureIgnoreCase); var tracer = DedupManifestArtifactClientFactory.CreateArtifactsTracer(verbose, (str) => context.Output(str)); var keepUntulRef = new KeepUntilBlobReference(DateTime.UtcNow.AddDays(1)); var uploadSession = _dedupClient.CreateUploadSession(keepUntulRef, tracer, FileSystem.Instance); // Upload the chunks var uploadRecord = _blobTelemetry.CreateRecord <BuildArtifactActionRecord>((level, uri, type) => new BuildArtifactActionRecord(level, uri, type, nameof(UploadAsync), context)); await _blobTelemetry.MeasureActionAsync( record : uploadRecord, actionAsync : async() => await AsyncHttpRetryHelper.InvokeAsync( async() => { return(await uploadSession.UploadAsync(rootNode, new Dictionary <DedupIdentifier, string>() { [dedupId] = itemPath }, cancellationToken)); }, maxRetries: 3, tracer: tracer, canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times cancellationToken: cancellationToken, continueOnCapturedContext: false) ); return(dedupId, rootNode.TransitiveContentBytes); }
internal async Task UploadAsync( AgentTaskPluginExecutionContext context, Fingerprint fingerprint, string path, CancellationToken cancellationToken, string preferredContentFormat = ContentFormatConstants.Files) { VssConnection connection = context.VssConnection; BlobStoreClientTelemetry clientTelemetry; DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry); PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection); using (clientTelemetry) { // Check if the key exists. PipelineCacheActionRecord cacheRecordGet = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context)); PipelineCacheArtifact getResult = await pipelineCacheClient.GetPipelineCacheArtifactAsync(new [] { fingerprint }, cancellationToken, cacheRecordGet); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecordGet); //If cache exists, return. if (getResult != null) { context.Output($"Cache with fingerprint `{getResult.Fingerprint}` already exists."); return; } string uploadPath = await this.GetUploadPathAsync(preferredContentFormat, context, path, cancellationToken); //Upload the pipeline artifact. PipelineCacheActionRecord uploadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, nameof(dedupManifestClient.PublishAsync), context)); PublishResult result = await clientTelemetry.MeasureActionAsync( record : uploadRecord, actionAsync : async() => { return(await dedupManifestClient.PublishAsync(uploadPath, cancellationToken)); }); CreatePipelineCacheArtifactContract options = new CreatePipelineCacheArtifactContract { Fingerprint = fingerprint, RootId = result.RootId, ManifestId = result.ManifestId, ProofNodes = result.ProofNodes.ToArray(), ContentFormat = preferredContentFormat }; // delete archive file if it's tar. if (string.Equals(preferredContentFormat, ContentFormatConstants.SingleTar, StringComparison.OrdinalIgnoreCase)) { try { if (File.Exists(uploadPath)) { File.Delete(uploadPath); } } catch { } } // Cache the artifact PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.SaveCache, context)); CreateStatus status = await pipelineCacheClient.CreatePipelineCacheArtifactAsync(options, cancellationToken, cacheRecord); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: uploadRecord); context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord); context.Output("Saved item."); } }
internal async Task DownloadAsync( AgentTaskPluginExecutionContext context, Fingerprint[] fingerprints, string path, string cacheHitVariable, CancellationToken cancellationToken) { VssConnection connection = context.VssConnection; BlobStoreClientTelemetry clientTelemetry; DedupManifestArtifactClient dedupManifestClient = DedupManifestArtifactClientFactory.CreateDedupManifestClient(context, connection, cancellationToken, out clientTelemetry); PipelineCacheClient pipelineCacheClient = this.CreateClient(clientTelemetry, context, connection); using (clientTelemetry) { PipelineCacheActionRecord cacheRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, PipelineArtifactConstants.RestoreCache, context)); PipelineCacheArtifact result = await pipelineCacheClient.GetPipelineCacheArtifactAsync(fingerprints, cancellationToken, cacheRecord); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: cacheRecord); if (result != null) { context.Output($"Entry found at fingerprint: `{result.Fingerprint.ToString()}`"); context.Verbose($"Manifest ID is: {result.ManifestId.ValueString}"); PipelineCacheActionRecord downloadRecord = clientTelemetry.CreateRecord <PipelineCacheActionRecord>((level, uri, type) => new PipelineCacheActionRecord(level, uri, type, nameof(DownloadAsync), context)); await clientTelemetry.MeasureActionAsync( record : downloadRecord, actionAsync : async() => { await this.DownloadPipelineCacheAsync(context, dedupManifestClient, result.ManifestId, path, result.ContentFormat, cancellationToken); }); // Send results to CustomerIntelligence context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineCache, record: downloadRecord); context.Output("Cache restored."); } if (!string.IsNullOrEmpty(cacheHitVariable)) { if (result == null) { context.SetVariable(cacheHitVariable, "false"); } else { context.Verbose($"Exact fingerprint: `{result.Fingerprint.ToString()}`"); bool foundExact = false; foreach (var fingerprint in fingerprints) { context.Verbose($"This fingerprint: `{fingerprint.ToString()}`"); if (fingerprint == result.Fingerprint || result.Fingerprint.Segments.Length == 1 && result.Fingerprint.Segments.Single() == fingerprint.SummarizeForV1()) { foundExact = true; break; } } context.SetVariable(cacheHitVariable, foundExact ? "true" : "inexact"); } } } }
public static async Task <(List <BlobFileInfo> fileDedupIds, ulong length)> UploadBatchToBlobstore( bool verbose, IReadOnlyList <string> itemPaths, Func <TelemetryInformationLevel, Uri, string, BlobStoreTelemetryRecord> telemetryRecordFactory, Action <string> traceOutput, DedupStoreClient dedupClient, BlobStoreClientTelemetry clientTelemetry, CancellationToken cancellationToken, bool enableReporting = false) { // Create chunks and identifier traceOutput(StringUtil.Loc("BuildingFileTree")); var fileNodes = await GenerateHashes(itemPaths, cancellationToken); var rootNode = CreateNodeToUpload(fileNodes.Where(x => x.Success).Select(y => y.Node)); // If there are multiple paths to one DedupId (duplicate files) // take the last one var fileDedupIds = new Dictionary <DedupIdentifier, string>(); foreach (var file in fileNodes.Where(x => x.Success)) { // ChunkHelper uses 64k block default size var dedupId = file.Node.GetDedupIdentifier(HashType.Dedup64K); fileDedupIds[dedupId] = file.Path; } // Setup upload session to keep file for at mimimum one day // Blobs will need to be associated with the server with an ID ref otherwise they will be // garbage collected after one day var tracer = DedupManifestArtifactClientFactory.CreateArtifactsTracer(verbose, traceOutput); var keepUntilRef = new KeepUntilBlobReference(DateTime.UtcNow.AddDays(1)); var uploadSession = dedupClient.CreateUploadSession(keepUntilRef, tracer, FileSystem.Instance); using (var reportingCancelSrc = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) { // Log stats Task reportingTask = null; if (enableReporting) { reportingTask = StartReportingTask(traceOutput, (long)rootNode.TransitiveContentBytes, uploadSession, reportingCancelSrc); } // Upload the chunks var uploadRecord = clientTelemetry.CreateRecord <BlobStoreTelemetryRecord>(telemetryRecordFactory); await clientTelemetry.MeasureActionAsync( record : uploadRecord, actionAsync : async() => await AsyncHttpRetryHelper.InvokeAsync( async() => { await uploadSession.UploadAsync(rootNode, fileDedupIds, cancellationToken); return(uploadSession.UploadStatistics); }, maxRetries: 3, tracer: tracer, canRetryDelegate: e => true, // this isn't great, but failing on upload stinks, so just try a couple of times cancellationToken: cancellationToken, continueOnCapturedContext: false) ); if (enableReporting) { reportingCancelSrc.Cancel(); await reportingTask; } } return(fileNodes, rootNode.TransitiveContentBytes); }