private async Task <BoolResult> GetFileWithDedupAsync(Context context, ContentHash contentHash, string path, CancellationToken cts)
{
VstsBlobIdentifier blobId = ToVstsBlobIdentifier(contentHash.ToBlobIdentifier());
VstsDedupIdentifier dedupId = blobId.ToDedupIdentifier();
try
{
await TryGatedArtifactOperationAsync <Object>(
context,
contentHash.ToString(),
"DownloadToFileAsync",
async innerCts =>
{
await DedupStoreClient.DownloadToFileAsync(dedupId, path, null, null, EdgeCache.Allowed, innerCts);
return(null);
}, cts);
}
catch (NullReferenceException) // Null reference thrown when DedupIdentifier doesn't exist in VSTS.
{
return(new BoolResult("DedupIdentifier not found."));
}
catch (Exception ex)
{
return(new BoolResult(ex));
}
return(BoolResult.Success);
}
/// <summary>
/// Attempt to update expiry of all children. Pin parent node if all children were extended successfully.
/// </summary>
private async Task <PinResult> TryPinChildrenAsync(Context context, VstsDedupIdentifier parentNode, IEnumerable <VstsDedupIdentifier> dedupIdentifiers, CancellationToken cts)
{
var chunks = new List <VstsDedupIdentifier>();
var nodes = new List <VstsDedupIdentifier>();
foreach (var id in dedupIdentifiers)
{
if (id.AlgorithmId == Hashing.ChunkDedupIdentifier.ChunkAlgorithmId)
{
chunks.Add(id);
}
else
{
nodes.Add(id);
}
}
// Attempt to save all children.
Tracer.Debug(context, $"Pinning children: nodes=[{string.Join(",", nodes.Select(x => x.ValueString))}] chunks=[{string.Join(",", chunks.Select(x => x.ValueString))}]");
var result = await TryPinNodesAsync(context, nodes, cts) & await TryPinChunksAsync(context, chunks, cts);
if (result == PinResult.Success)
{
// If all children are saved, pin parent.
result = await TryPinNodeAsync(context, parentNode, cts);
}
return(result);
}
/// <summary>
/// Attempt to update expiry of all children. Pin parent node if all children were extended successfully.
/// </summary>
private async Task <PinResult> TryPinChildrenAsync(OperationContext context, VstsDedupIdentifier parentNode, IEnumerable <VstsDedupIdentifier> dedupIdentifiers)
{
var chunks = new List <VstsDedupIdentifier>();
var nodes = new List <VstsDedupIdentifier>();
foreach (var id in dedupIdentifiers)
{
if (id.AlgorithmId == Hashing.ChunkDedupIdentifier.ChunkAlgorithmId)
{
chunks.Add(id);
}
else if (((NodeAlgorithmId)id.AlgorithmId).IsValidNode())
{
nodes.Add(id);
}
else
{
throw new InvalidOperationException($"Unknown dedup algorithm id detected for dedup {id.ValueString} : {id.AlgorithmId}");
}
}
// Attempt to save all children.
Tracer.Debug(context, $"Pinning children: nodes=[{string.Join(",", nodes.Select(x => x.ValueString))}] chunks=[{string.Join(",", chunks.Select(x => x.ValueString))}]");
var result = await TryPinNodesAsync(context, nodes) & await TryPinChunksAsync(context, chunks);
if (result == PinResult.Success)
{
// If all children are saved, pin parent.
result = await TryPinNodeAsync(context, parentNode);
}
return(result);
}
/// <summary>
/// Updates expiry of single node in DedupStore if
/// 1) Node exists
/// 2) All children exist and have sufficient TTL
/// If children have insufficient TTL, attempt to extend the expiry of all children before pinning.
/// </summary>
private async Task <PinResult> TryPinNodeAsync(Context context, VstsDedupIdentifier dedupId, CancellationToken cts)
{
TryReferenceNodeResponse referenceResult;
try
{
referenceResult = await TryGatedArtifactOperationAsync(
context,
dedupId.ValueString,
"TryKeepUntilReferenceNode",
innerCts => DedupStoreClient.Client.TryKeepUntilReferenceNodeAsync(dedupId.CastToNodeDedupIdentifier(), new KeepUntilBlobReference(EndDateTime), null, innerCts),
cts);
}
catch (DedupNotFoundException)
{
// When VSTS processes response, throws exception when node doesn't exist.
referenceResult = new TryReferenceNodeResponse(new DedupNodeNotFound());
}
catch (Exception ex)
{
return(new PinResult(ex));
}
PinResult pinResult = PinResult.ContentNotFound;
referenceResult.Match(
(notFound) =>
{
// Root node has expired.
},
async(needAction) =>
{
pinResult = await TryPinChildrenAsync(context, dedupId, needAction.InsufficientKeepUntil, cts);
},
(added) =>
{
pinResult = PinResult.Success;
});
return(pinResult);
}
/// <summary>
/// Updates expiry of single chunk in DedupStore if it exists.
/// </summary>
private async Task <PinResult> TryPinChunkAsync(OperationContext context, VstsDedupIdentifier dedupId)
{
try
{
var receipt = await TryGatedArtifactOperationAsync(
context,
dedupId.ValueString,
"TryKeepUntilReferenceChunk",
innerCts => DedupStoreClient.Client.TryKeepUntilReferenceChunkAsync(dedupId.CastToChunkDedupIdentifier(), new KeepUntilBlobReference(EndDateTime), innerCts));
if (receipt == null)
{
return(PinResult.ContentNotFound);
}
return(PinResult.Success);
}
catch (Exception ex)
{
return(new PinResult(ex));
}
}
/// <summary>
/// Checks the current keepUntil of a node. Returns null if the node is not found.
/// </summary>
protected async Task <Result <DateTime?> > CheckNodeKeepUntilAsync(OperationContext context, VstsDedupIdentifier dedupId)
{
TryReferenceNodeResponse referenceResult;
try
{
// Pinning with keepUntil of now means that, if the content is available, the call will always succeed.
referenceResult = await TryGatedArtifactOperationAsync(
context,
dedupId.ValueString,
"TryKeepUntilReferenceNode",
innerCts => DedupStoreClient.Client.TryKeepUntilReferenceNodeAsync(dedupId.CastToNodeDedupIdentifier(), new KeepUntilBlobReference(DateTime.UtcNow), null, innerCts));
}
catch (Exception ex)
{
return(new Result <DateTime?>(ex));
}
DateTime?keepUntil = null;
referenceResult.Match(
(notFound) => { /* Do nothing */ },
(needAction) =>
{
// For the reason explained above, this case where children need to be pinned should never happen.
// However, as a best approximation, we take the min of all the children, which always outlive the parent.
keepUntil = needAction.Receipts.Select(r => r.Value.KeepUntil.KeepUntil).Min();
},
(added) =>
{
keepUntil = added.Receipts[dedupId].KeepUntil.KeepUntil;
});
return(new Result <DateTime?>(keepUntil, isNullAllowed: true));
}
/// <inheritdoc />
protected override async Task <PinResult> PinCoreAsync(
OperationContext context, ContentHash contentHash, UrgencyHint urgencyHint, Counter retryCounter)
{
if (!contentHash.HashType.IsValidDedup())
{
return(new PinResult($"DedupStore client requires a HashType that supports dedup. Given hash type: {contentHash.HashType}."));
}
var pinResult = CheckPinInMemory(contentHash);
if (pinResult.Succeeded)
{
return(pinResult);
}
VstsBlobIdentifier blobId = ToVstsBlobIdentifier(contentHash.ToBlobIdentifier());
VstsDedupIdentifier dedupId = blobId.ToDedupIdentifier();
if (dedupId.AlgorithmId == Hashing.ChunkDedupIdentifier.ChunkAlgorithmId)
{
// No need to optimize since pinning a chunk is always a fast operation.
return(await PinImplAsync(context, contentHash));
}
// Since pinning the whole tree can be an expensive operation, we have optimized how we call it. Depending on the current
// keepUntil of the root node, which is unexpensive to check, the operation will behave differently:
// The pin operation will be ignored if it is greater than ignorePinThreshold, to reduce amount of calls
// The pin operation will be inlined if it is lower than pinInlineThreshold, to make sure that we don't try to use
// content that we pin in the background but has expired before we could complete the pin.
// The pin operation will be done asynchronously and will return success otherwise. Most calls should follow this
// behavior, to avoid waiting on a potentially long operation. We're confident returning a success because we
// know that the content is there even though we still have to extend it's keepUntil
var keepUntilResult = await CheckNodeKeepUntilAsync(context, dedupId);
if (!keepUntilResult.Succeeded)
{
// Returned a service error. Fail fast.
return(new PinResult(keepUntilResult));
}
else if (!keepUntilResult.Value.HasValue)
{
// Content not found.
return(new PinResult(PinResult.ResultCode.ContentNotFound));
}
var timeLeft = keepUntilResult.Value.Value - DateTime.UtcNow;
if (timeLeft > _ignorePinThreshold)
{
Tracer.Debug(context, $"Pin was skipped because keepUntil has remaining time [{timeLeft}] that is greater than ignorePinThreshold=[{_ignorePinThreshold}]");
_dedupCounters[Counters.PinIgnored].Increment();
return(PinResult.Success);
}
var pinTask = PinImplAsync(context, contentHash);
if (timeLeft < _pinInlineThreshold)
{
Tracer.Debug(context, $"Pin inlined because keepUntil has remaining time [{timeLeft}] that is less than pinInlineThreshold=[{_pinInlineThreshold}]");
_dedupCounters[Counters.PinInlined].Increment();
return(await pinTask);
}
pinTask.FireAndForget(context);
return(PinResult.Success);
}
