private async Task <BoolResult> GetFileWithDedupAsync(Context context, ContentHash contentHash, string path, CancellationToken cts) { VstsBlobIdentifier blobId = ToVstsBlobIdentifier(contentHash.ToBlobIdentifier()); VstsDedupIdentifier dedupId = blobId.ToDedupIdentifier(); try { await TryGatedArtifactOperationAsync <Object>( context, contentHash.ToString(), "DownloadToFileAsync", async innerCts => { await DedupStoreClient.DownloadToFileAsync(dedupId, path, null, null, EdgeCache.Allowed, innerCts); return(null); }, cts); } catch (NullReferenceException) // Null reference thrown when DedupIdentifier doesn't exist in VSTS. { return(new BoolResult("DedupIdentifier not found.")); } catch (Exception ex) { return(new BoolResult(ex)); } return(BoolResult.Success); }
/// <summary> /// Attempt to update expiry of all children. Pin parent node if all children were extended successfully. /// </summary> private async Task <PinResult> TryPinChildrenAsync(Context context, VstsDedupIdentifier parentNode, IEnumerable <VstsDedupIdentifier> dedupIdentifiers, CancellationToken cts) { var chunks = new List <VstsDedupIdentifier>(); var nodes = new List <VstsDedupIdentifier>(); foreach (var id in dedupIdentifiers) { if (id.AlgorithmId == Hashing.ChunkDedupIdentifier.ChunkAlgorithmId) { chunks.Add(id); } else { nodes.Add(id); } } // Attempt to save all children. Tracer.Debug(context, $"Pinning children: nodes=[{string.Join(",", nodes.Select(x => x.ValueString))}] chunks=[{string.Join(",", chunks.Select(x => x.ValueString))}]"); var result = await TryPinNodesAsync(context, nodes, cts) & await TryPinChunksAsync(context, chunks, cts); if (result == PinResult.Success) { // If all children are saved, pin parent. result = await TryPinNodeAsync(context, parentNode, cts); } return(result); }
/// <summary> /// Attempt to update expiry of all children. Pin parent node if all children were extended successfully. /// </summary> private async Task <PinResult> TryPinChildrenAsync(OperationContext context, VstsDedupIdentifier parentNode, IEnumerable <VstsDedupIdentifier> dedupIdentifiers) { var chunks = new List <VstsDedupIdentifier>(); var nodes = new List <VstsDedupIdentifier>(); foreach (var id in dedupIdentifiers) { if (id.AlgorithmId == Hashing.ChunkDedupIdentifier.ChunkAlgorithmId) { chunks.Add(id); } else if (((NodeAlgorithmId)id.AlgorithmId).IsValidNode()) { nodes.Add(id); } else { throw new InvalidOperationException($"Unknown dedup algorithm id detected for dedup {id.ValueString} : {id.AlgorithmId}"); } } // Attempt to save all children. Tracer.Debug(context, $"Pinning children: nodes=[{string.Join(",", nodes.Select(x => x.ValueString))}] chunks=[{string.Join(",", chunks.Select(x => x.ValueString))}]"); var result = await TryPinNodesAsync(context, nodes) & await TryPinChunksAsync(context, chunks); if (result == PinResult.Success) { // If all children are saved, pin parent. result = await TryPinNodeAsync(context, parentNode); } return(result); }
/// <summary> /// Updates expiry of single node in DedupStore if /// 1) Node exists /// 2) All children exist and have sufficient TTL /// If children have insufficient TTL, attempt to extend the expiry of all children before pinning. /// </summary> private async Task <PinResult> TryPinNodeAsync(Context context, VstsDedupIdentifier dedupId, CancellationToken cts) { TryReferenceNodeResponse referenceResult; try { referenceResult = await TryGatedArtifactOperationAsync( context, dedupId.ValueString, "TryKeepUntilReferenceNode", innerCts => DedupStoreClient.Client.TryKeepUntilReferenceNodeAsync(dedupId.CastToNodeDedupIdentifier(), new KeepUntilBlobReference(EndDateTime), null, innerCts), cts); } catch (DedupNotFoundException) { // When VSTS processes response, throws exception when node doesn't exist. referenceResult = new TryReferenceNodeResponse(new DedupNodeNotFound()); } catch (Exception ex) { return(new PinResult(ex)); } PinResult pinResult = PinResult.ContentNotFound; referenceResult.Match( (notFound) => { // Root node has expired. }, async(needAction) => { pinResult = await TryPinChildrenAsync(context, dedupId, needAction.InsufficientKeepUntil, cts); }, (added) => { pinResult = PinResult.Success; }); return(pinResult); }
/// <summary> /// Updates expiry of single chunk in DedupStore if it exists. /// </summary> private async Task <PinResult> TryPinChunkAsync(OperationContext context, VstsDedupIdentifier dedupId) { try { var receipt = await TryGatedArtifactOperationAsync( context, dedupId.ValueString, "TryKeepUntilReferenceChunk", innerCts => DedupStoreClient.Client.TryKeepUntilReferenceChunkAsync(dedupId.CastToChunkDedupIdentifier(), new KeepUntilBlobReference(EndDateTime), innerCts)); if (receipt == null) { return(PinResult.ContentNotFound); } return(PinResult.Success); } catch (Exception ex) { return(new PinResult(ex)); } }
/// <summary> /// Checks the current keepUntil of a node. Returns null if the node is not found. /// </summary> protected async Task <Result <DateTime?> > CheckNodeKeepUntilAsync(OperationContext context, VstsDedupIdentifier dedupId) { TryReferenceNodeResponse referenceResult; try { // Pinning with keepUntil of now means that, if the content is available, the call will always succeed. referenceResult = await TryGatedArtifactOperationAsync( context, dedupId.ValueString, "TryKeepUntilReferenceNode", innerCts => DedupStoreClient.Client.TryKeepUntilReferenceNodeAsync(dedupId.CastToNodeDedupIdentifier(), new KeepUntilBlobReference(DateTime.UtcNow), null, innerCts)); } catch (Exception ex) { return(new Result <DateTime?>(ex)); } DateTime?keepUntil = null; referenceResult.Match( (notFound) => { /* Do nothing */ }, (needAction) => { // For the reason explained above, this case where children need to be pinned should never happen. // However, as a best approximation, we take the min of all the children, which always outlive the parent. keepUntil = needAction.Receipts.Select(r => r.Value.KeepUntil.KeepUntil).Min(); }, (added) => { keepUntil = added.Receipts[dedupId].KeepUntil.KeepUntil; }); return(new Result <DateTime?>(keepUntil, isNullAllowed: true)); }
/// <inheritdoc /> protected override async Task <PinResult> PinCoreAsync( OperationContext context, ContentHash contentHash, UrgencyHint urgencyHint, Counter retryCounter) { if (!contentHash.HashType.IsValidDedup()) { return(new PinResult($"DedupStore client requires a HashType that supports dedup. Given hash type: {contentHash.HashType}.")); } var pinResult = CheckPinInMemory(contentHash); if (pinResult.Succeeded) { return(pinResult); } VstsBlobIdentifier blobId = ToVstsBlobIdentifier(contentHash.ToBlobIdentifier()); VstsDedupIdentifier dedupId = blobId.ToDedupIdentifier(); if (dedupId.AlgorithmId == Hashing.ChunkDedupIdentifier.ChunkAlgorithmId) { // No need to optimize since pinning a chunk is always a fast operation. return(await PinImplAsync(context, contentHash)); } // Since pinning the whole tree can be an expensive operation, we have optimized how we call it. Depending on the current // keepUntil of the root node, which is unexpensive to check, the operation will behave differently: // The pin operation will be ignored if it is greater than ignorePinThreshold, to reduce amount of calls // The pin operation will be inlined if it is lower than pinInlineThreshold, to make sure that we don't try to use // content that we pin in the background but has expired before we could complete the pin. // The pin operation will be done asynchronously and will return success otherwise. Most calls should follow this // behavior, to avoid waiting on a potentially long operation. We're confident returning a success because we // know that the content is there even though we still have to extend it's keepUntil var keepUntilResult = await CheckNodeKeepUntilAsync(context, dedupId); if (!keepUntilResult.Succeeded) { // Returned a service error. Fail fast. return(new PinResult(keepUntilResult)); } else if (!keepUntilResult.Value.HasValue) { // Content not found. return(new PinResult(PinResult.ResultCode.ContentNotFound)); } var timeLeft = keepUntilResult.Value.Value - DateTime.UtcNow; if (timeLeft > _ignorePinThreshold) { Tracer.Debug(context, $"Pin was skipped because keepUntil has remaining time [{timeLeft}] that is greater than ignorePinThreshold=[{_ignorePinThreshold}]"); _dedupCounters[Counters.PinIgnored].Increment(); return(PinResult.Success); } var pinTask = PinImplAsync(context, contentHash); if (timeLeft < _pinInlineThreshold) { Tracer.Debug(context, $"Pin inlined because keepUntil has remaining time [{timeLeft}] that is less than pinInlineThreshold=[{_pinInlineThreshold}]"); _dedupCounters[Counters.PinInlined].Increment(); return(await pinTask); } pinTask.FireAndForget(context); return(PinResult.Success); }