private async Task <BoolResult> PutLazyStreamAsync(
OperationContext context,
ContentHash contentHash,
Stream stream,
UrgencyHint urgencyHint)
{
var pinResult = await PinAsync(context, contentHash, context.Token, urgencyHint);
if (pinResult.Code == PinResult.ResultCode.Success)
{
return(BoolResult.Success);
}
// Puts are effectively implicitly pinned regardless of configuration.
try
{
var endDateTime = DateTime.UtcNow + TimeToKeepContent;
await BlobStoreHttpClient.UploadAndReferenceBlobWithRetriesAsync(
ToVstsBlobIdentifier(contentHash.ToBlobIdentifier()),
stream,
new BlobReference(endDateTime),
context,
context.Token);
return(BoolResult.Success);
}
catch (Exception ex)
{
return(new BoolResult(ex));
}
}
private async Task <BoolResult> GetFileWithDedupAsync(Context context, ContentHash contentHash, string path, CancellationToken cts)
{
VstsBlobIdentifier blobId = ToVstsBlobIdentifier(contentHash.ToBlobIdentifier());
VstsDedupIdentifier dedupId = blobId.ToDedupIdentifier();
try
{
await TryGatedArtifactOperationAsync <Object>(
context,
contentHash.ToString(),
"DownloadToFileAsync",
async innerCts =>
{
await DedupStoreClient.DownloadToFileAsync(dedupId, path, null, null, EdgeCache.Allowed, innerCts);
return(null);
}, cts);
}
catch (NullReferenceException) // Null reference thrown when DedupIdentifier doesn't exist in VSTS.
{
return(new BoolResult("DedupIdentifier not found."));
}
catch (Exception ex)
{
return(new BoolResult(ex));
}
return(BoolResult.Success);
}
private async Task <PinResult> PinImplAsync(OperationContext context, ContentHash contentHash)
{
try
{
PinResult pinResult;
var dedupId = ToVstsBlobIdentifier(contentHash.ToBlobIdentifier()).ToDedupIdentifier();
if (dedupId.AlgorithmId == Hashing.ChunkDedupIdentifier.ChunkAlgorithmId)
{
pinResult = await TryPinChunkAsync(context, dedupId);
}
else if (((NodeAlgorithmId)dedupId.AlgorithmId).IsValidNode())
{
pinResult = await TryPinNodeAsync(context, dedupId);
}
else
{
throw new InvalidOperationException($"Unknown dedup algorithm id detected for dedup {dedupId.ValueString} : {dedupId.AlgorithmId}");
}
if (pinResult.Succeeded)
{
_counters[BackingContentStore.SessionCounters.PinSatisfiedFromRemote].Increment();
BackingContentStoreExpiryCache.Instance.AddExpiry(contentHash, EndDateTime);
}
return(pinResult);
}
catch (Exception ex)
{
return(new PinResult(ex));
}
}
private async Task <Uri?> GetUriAsync(OperationContext context, ContentHash contentHash)
{
if (!DownloadUriCache.Instance.TryGetDownloadUri(contentHash, out var uri))
{
_blobCounters[Counters.VstsDownloadUriFetchedFromRemote].Increment();
var blobId = contentHash.ToBlobIdentifier();
var mappings = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync(
context,
"GetStreamInternal",
innerCts => BlobStoreHttpClient.GetDownloadUrisAsync(
new[] { blobId },
EdgeCache.NotAllowed,
cancellationToken: innerCts),
context.Token).ConfigureAwait(false);
if (mappings == null || !mappings.TryGetValue(blobId, out uri))
{
return(null);
}
DownloadUriCache.Instance.AddDownloadUri(contentHash, uri);
}
else
{
_blobCounters[Counters.VstsDownloadUriFetchedInMemory].Increment();
}
return(uri.NotNullUri);
}
private async Task <Stream?> GetStreamInternalAsync(OperationContext context, ContentHash contentHash, int?overrideStreamMinimumReadSizeInBytes)
{
Uri?azureBlobUri = default;
try
{
if (_downloadBlobsThroughBlobStore)
{
return(await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync(
context,
"GetStreamInternalThroughBlobStore",
innerCts => BlobStoreHttpClient.GetBlobAsync(ToVstsBlobIdentifier(contentHash.ToBlobIdentifier()), cancellationToken: innerCts),
context.Token).ConfigureAwait(false));
}
else
{
if (!DownloadUriCache.Instance.TryGetDownloadUri(contentHash, out var uri))
{
_blobCounters[Counters.VstsDownloadUriFetchedFromRemote].Increment();
var blobId = contentHash.ToBlobIdentifier();
var mappings = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync(
context,
"GetStreamInternal",
innerCts => BlobStoreHttpClient.GetDownloadUrisAsync(
new[] { ToVstsBlobIdentifier(blobId) },
EdgeCache.NotAllowed,
cancellationToken: innerCts),
context.Token).ConfigureAwait(false);
if (mappings == null || !mappings.TryGetValue(ToVstsBlobIdentifier(blobId), out uri))
{
return(null);
}
DownloadUriCache.Instance.AddDownloadUri(contentHash, uri);
}
else
{
_blobCounters[Counters.VstsDownloadUriFetchedInMemory].Increment();
}
azureBlobUri = uri.NotNullUri;
return(await GetStreamThroughAzureBlobs(
uri.NotNullUri,
overrideStreamMinimumReadSizeInBytes,
_parallelSegmentDownloadConfig.SegmentDownloadTimeout,
context.Token).ConfigureAwait(false));
}
}
catch (Exception e)
{
TraceException(context, contentHash, azureBlobUri, e);
throw;
}
}
/// <inheritdoc />
public async Task <PinResult> PinAsync(
Context context, ContentHash contentHash, CancellationToken cts, UrgencyHint urgencyHint = UrgencyHint.Nominal)
{
if (contentHash.HashType != RequiredHashType)
{
return(new PinResult($"DedupStore client requires {RequiredHashType}. Cannot take HashType '{contentHash.HashType}'."));
}
Stopwatch sw = Stopwatch.StartNew();
try
{
Tracer.PinBulkStart(context, new[] { contentHash });
var pinResult = CheckPinInMemory(contentHash);
if (pinResult.Succeeded)
{
return(pinResult);
}
var dedupId = ToVstsBlobIdentifier(contentHash.ToBlobIdentifier()).ToDedupIdentifier();
if (dedupId.AlgorithmId == Hashing.ChunkDedupIdentifier.ChunkAlgorithmId)
{
pinResult = await TryPinChunkAsync(context, dedupId, cts);
}
else
{
pinResult = await TryPinNodeAsync(context, dedupId, cts);
}
if (pinResult.Succeeded)
{
BackingContentStoreExpiryCache.Instance.AddExpiry(contentHash, EndDateTime);
}
return(pinResult);
}
catch (Exception ex)
{
return(new PinResult(ex));
}
finally
{
Tracer.PinBulkStop(context, sw.Elapsed);
}
}
private async Task <Stream> GetStreamInternalAsync(Context context, ContentHash contentHash, int?overrideStreamMinimumReadSizeInBytes, CancellationToken cts)
{
if (_downloadBlobsThroughBlobStore)
{
return(await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync(
context,
"GetStreamInternalThroughBlobStore",
innerCts => BlobStoreHttpClient.GetBlobAsync(ToVstsBlobIdentifier(contentHash.ToBlobIdentifier()), cancellationToken: innerCts),
cts).ConfigureAwait(false));
}
else
{
PreauthenticatedUri uri;
if (!DownloadUriCache.Instance.TryGetDownloadUri(contentHash, out uri))
{
Tracer.RecordDownloadUriFetchedFromRemote();
BlobIdentifier blobId = contentHash.ToBlobIdentifier();
IDictionary <VstsBlobIdentifier, PreauthenticatedUri> mappings = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync(
context,
"GetStreamInternal",
innerCts => BlobStoreHttpClient.GetDownloadUrisAsync(
new[] { ToVstsBlobIdentifier(blobId) },
EdgeCache.NotAllowed,
cancellationToken: innerCts),
cts).ConfigureAwait(false);
if (mappings == null || !mappings.TryGetValue(ToVstsBlobIdentifier(blobId), out uri))
{
return(null);
}
DownloadUriCache.Instance.AddDownloadUri(contentHash, uri);
}
else
{
Tracer.RecordDownloadUriFetchedFromCache();
}
return(await GetStreamThroughAzureBlobs(
uri.NotNullUri,
overrideStreamMinimumReadSizeInBytes,
_parallelSegmentDownloadConfig.SegmentDownloadTimeout,
cts).ConfigureAwait(false));
}
}
/// <inheritdoc />
protected override async Task <PinResult> PinCoreAsync(
OperationContext context, ContentHash contentHash, UrgencyHint urgencyHint, Counter retryCounter)
{
if (contentHash.HashType != RequiredHashType)
{
return(new PinResult($"DedupStore client requires {RequiredHashType}. Cannot take HashType '{contentHash.HashType}'."));
}
try
{
var pinResult = CheckPinInMemory(contentHash);
if (pinResult.Succeeded)
{
return(pinResult);
}
var dedupId = ToVstsBlobIdentifier(contentHash.ToBlobIdentifier()).ToDedupIdentifier();
if (dedupId.AlgorithmId == Hashing.ChunkDedupIdentifier.ChunkAlgorithmId)
{
pinResult = await TryPinChunkAsync(context, dedupId);
}
else
{
pinResult = await TryPinNodeAsync(context, dedupId);
}
if (pinResult.Succeeded)
{
_counters[BackingContentStore.SessionCounters.PinSatisfiedFromRemote].Increment();
BackingContentStoreExpiryCache.Instance.AddExpiry(contentHash, EndDateTime);
}
return(pinResult);
}
catch (Exception ex)
{
return(new PinResult(ex));
}
}
/// <inheritdoc />
protected override async Task <PinResult> PinCoreAsync(
OperationContext context, ContentHash contentHash, UrgencyHint urgencyHint, Counter retryCounter)
{
if (!contentHash.HashType.IsValidDedup())
{
return(new PinResult($"DedupStore client requires a HashType that supports dedup. Given hash type: {contentHash.HashType}."));
}
var pinResult = CheckPinInMemory(contentHash);
if (pinResult.Succeeded)
{
return(pinResult);
}
VstsBlobIdentifier blobId = ToVstsBlobIdentifier(contentHash.ToBlobIdentifier());
VstsDedupIdentifier dedupId = blobId.ToDedupIdentifier();
if (dedupId.AlgorithmId == Hashing.ChunkDedupIdentifier.ChunkAlgorithmId)
{
// No need to optimize since pinning a chunk is always a fast operation.
return(await PinImplAsync(context, contentHash));
}
// Since pinning the whole tree can be an expensive operation, we have optimized how we call it. Depending on the current
// keepUntil of the root node, which is unexpensive to check, the operation will behave differently:
// The pin operation will be ignored if it is greater than ignorePinThreshold, to reduce amount of calls
// The pin operation will be inlined if it is lower than pinInlineThreshold, to make sure that we don't try to use
// content that we pin in the background but has expired before we could complete the pin.
// The pin operation will be done asynchronously and will return success otherwise. Most calls should follow this
// behavior, to avoid waiting on a potentially long operation. We're confident returning a success because we
// know that the content is there even though we still have to extend it's keepUntil
var keepUntilResult = await CheckNodeKeepUntilAsync(context, dedupId);
if (!keepUntilResult.Succeeded)
{
// Returned a service error. Fail fast.
return(new PinResult(keepUntilResult));
}
else if (!keepUntilResult.Value.HasValue)
{
// Content not found.
return(new PinResult(PinResult.ResultCode.ContentNotFound));
}
var timeLeft = keepUntilResult.Value.Value - DateTime.UtcNow;
if (timeLeft > _ignorePinThreshold)
{
Tracer.Debug(context, $"Pin was skipped because keepUntil has remaining time [{timeLeft}] that is greater than ignorePinThreshold=[{_ignorePinThreshold}]");
_dedupCounters[Counters.PinIgnored].Increment();
return(PinResult.Success);
}
var pinTask = PinImplAsync(context, contentHash);
if (timeLeft < _pinInlineThreshold)
{
Tracer.Debug(context, $"Pin inlined because keepUntil has remaining time [{timeLeft}] that is less than pinInlineThreshold=[{_pinInlineThreshold}]");
_dedupCounters[Counters.PinInlined].Increment();
return(await pinTask);
}
pinTask.FireAndForget(context);
return(PinResult.Success);
}
