private async Task <BoolResult> PutLazyStreamAsync( OperationContext context, ContentHash contentHash, Stream stream, UrgencyHint urgencyHint) { var pinResult = await PinAsync(context, contentHash, context.Token, urgencyHint); if (pinResult.Code == PinResult.ResultCode.Success) { return(BoolResult.Success); } // Puts are effectively implicitly pinned regardless of configuration. try { var endDateTime = DateTime.UtcNow + TimeToKeepContent; await BlobStoreHttpClient.UploadAndReferenceBlobWithRetriesAsync( ToVstsBlobIdentifier(contentHash.ToBlobIdentifier()), stream, new BlobReference(endDateTime), context, context.Token); return(BoolResult.Success); } catch (Exception ex) { return(new BoolResult(ex)); } }
private async Task <BoolResult> GetFileWithDedupAsync(Context context, ContentHash contentHash, string path, CancellationToken cts) { VstsBlobIdentifier blobId = ToVstsBlobIdentifier(contentHash.ToBlobIdentifier()); VstsDedupIdentifier dedupId = blobId.ToDedupIdentifier(); try { await TryGatedArtifactOperationAsync <Object>( context, contentHash.ToString(), "DownloadToFileAsync", async innerCts => { await DedupStoreClient.DownloadToFileAsync(dedupId, path, null, null, EdgeCache.Allowed, innerCts); return(null); }, cts); } catch (NullReferenceException) // Null reference thrown when DedupIdentifier doesn't exist in VSTS. { return(new BoolResult("DedupIdentifier not found.")); } catch (Exception ex) { return(new BoolResult(ex)); } return(BoolResult.Success); }
private async Task <PinResult> PinImplAsync(OperationContext context, ContentHash contentHash) { try { PinResult pinResult; var dedupId = ToVstsBlobIdentifier(contentHash.ToBlobIdentifier()).ToDedupIdentifier(); if (dedupId.AlgorithmId == Hashing.ChunkDedupIdentifier.ChunkAlgorithmId) { pinResult = await TryPinChunkAsync(context, dedupId); } else if (((NodeAlgorithmId)dedupId.AlgorithmId).IsValidNode()) { pinResult = await TryPinNodeAsync(context, dedupId); } else { throw new InvalidOperationException($"Unknown dedup algorithm id detected for dedup {dedupId.ValueString} : {dedupId.AlgorithmId}"); } if (pinResult.Succeeded) { _counters[BackingContentStore.SessionCounters.PinSatisfiedFromRemote].Increment(); BackingContentStoreExpiryCache.Instance.AddExpiry(contentHash, EndDateTime); } return(pinResult); } catch (Exception ex) { return(new PinResult(ex)); } }
private async Task <Uri?> GetUriAsync(OperationContext context, ContentHash contentHash) { if (!DownloadUriCache.Instance.TryGetDownloadUri(contentHash, out var uri)) { _blobCounters[Counters.VstsDownloadUriFetchedFromRemote].Increment(); var blobId = contentHash.ToBlobIdentifier(); var mappings = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync( context, "GetStreamInternal", innerCts => BlobStoreHttpClient.GetDownloadUrisAsync( new[] { blobId }, EdgeCache.NotAllowed, cancellationToken: innerCts), context.Token).ConfigureAwait(false); if (mappings == null || !mappings.TryGetValue(blobId, out uri)) { return(null); } DownloadUriCache.Instance.AddDownloadUri(contentHash, uri); } else { _blobCounters[Counters.VstsDownloadUriFetchedInMemory].Increment(); } return(uri.NotNullUri); }
private async Task <Stream?> GetStreamInternalAsync(OperationContext context, ContentHash contentHash, int?overrideStreamMinimumReadSizeInBytes) { Uri?azureBlobUri = default; try { if (_downloadBlobsThroughBlobStore) { return(await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync( context, "GetStreamInternalThroughBlobStore", innerCts => BlobStoreHttpClient.GetBlobAsync(ToVstsBlobIdentifier(contentHash.ToBlobIdentifier()), cancellationToken: innerCts), context.Token).ConfigureAwait(false)); } else { if (!DownloadUriCache.Instance.TryGetDownloadUri(contentHash, out var uri)) { _blobCounters[Counters.VstsDownloadUriFetchedFromRemote].Increment(); var blobId = contentHash.ToBlobIdentifier(); var mappings = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync( context, "GetStreamInternal", innerCts => BlobStoreHttpClient.GetDownloadUrisAsync( new[] { ToVstsBlobIdentifier(blobId) }, EdgeCache.NotAllowed, cancellationToken: innerCts), context.Token).ConfigureAwait(false); if (mappings == null || !mappings.TryGetValue(ToVstsBlobIdentifier(blobId), out uri)) { return(null); } DownloadUriCache.Instance.AddDownloadUri(contentHash, uri); } else { _blobCounters[Counters.VstsDownloadUriFetchedInMemory].Increment(); } azureBlobUri = uri.NotNullUri; return(await GetStreamThroughAzureBlobs( uri.NotNullUri, overrideStreamMinimumReadSizeInBytes, _parallelSegmentDownloadConfig.SegmentDownloadTimeout, context.Token).ConfigureAwait(false)); } } catch (Exception e) { TraceException(context, contentHash, azureBlobUri, e); throw; } }
/// <inheritdoc /> public async Task <PinResult> PinAsync( Context context, ContentHash contentHash, CancellationToken cts, UrgencyHint urgencyHint = UrgencyHint.Nominal) { if (contentHash.HashType != RequiredHashType) { return(new PinResult($"DedupStore client requires {RequiredHashType}. Cannot take HashType '{contentHash.HashType}'.")); } Stopwatch sw = Stopwatch.StartNew(); try { Tracer.PinBulkStart(context, new[] { contentHash }); var pinResult = CheckPinInMemory(contentHash); if (pinResult.Succeeded) { return(pinResult); } var dedupId = ToVstsBlobIdentifier(contentHash.ToBlobIdentifier()).ToDedupIdentifier(); if (dedupId.AlgorithmId == Hashing.ChunkDedupIdentifier.ChunkAlgorithmId) { pinResult = await TryPinChunkAsync(context, dedupId, cts); } else { pinResult = await TryPinNodeAsync(context, dedupId, cts); } if (pinResult.Succeeded) { BackingContentStoreExpiryCache.Instance.AddExpiry(contentHash, EndDateTime); } return(pinResult); } catch (Exception ex) { return(new PinResult(ex)); } finally { Tracer.PinBulkStop(context, sw.Elapsed); } }
private async Task <Stream> GetStreamInternalAsync(Context context, ContentHash contentHash, int?overrideStreamMinimumReadSizeInBytes, CancellationToken cts) { if (_downloadBlobsThroughBlobStore) { return(await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync( context, "GetStreamInternalThroughBlobStore", innerCts => BlobStoreHttpClient.GetBlobAsync(ToVstsBlobIdentifier(contentHash.ToBlobIdentifier()), cancellationToken: innerCts), cts).ConfigureAwait(false)); } else { PreauthenticatedUri uri; if (!DownloadUriCache.Instance.TryGetDownloadUri(contentHash, out uri)) { Tracer.RecordDownloadUriFetchedFromRemote(); BlobIdentifier blobId = contentHash.ToBlobIdentifier(); IDictionary <VstsBlobIdentifier, PreauthenticatedUri> mappings = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync( context, "GetStreamInternal", innerCts => BlobStoreHttpClient.GetDownloadUrisAsync( new[] { ToVstsBlobIdentifier(blobId) }, EdgeCache.NotAllowed, cancellationToken: innerCts), cts).ConfigureAwait(false); if (mappings == null || !mappings.TryGetValue(ToVstsBlobIdentifier(blobId), out uri)) { return(null); } DownloadUriCache.Instance.AddDownloadUri(contentHash, uri); } else { Tracer.RecordDownloadUriFetchedFromCache(); } return(await GetStreamThroughAzureBlobs( uri.NotNullUri, overrideStreamMinimumReadSizeInBytes, _parallelSegmentDownloadConfig.SegmentDownloadTimeout, cts).ConfigureAwait(false)); } }
/// <inheritdoc /> protected override async Task <PinResult> PinCoreAsync( OperationContext context, ContentHash contentHash, UrgencyHint urgencyHint, Counter retryCounter) { if (contentHash.HashType != RequiredHashType) { return(new PinResult($"DedupStore client requires {RequiredHashType}. Cannot take HashType '{contentHash.HashType}'.")); } try { var pinResult = CheckPinInMemory(contentHash); if (pinResult.Succeeded) { return(pinResult); } var dedupId = ToVstsBlobIdentifier(contentHash.ToBlobIdentifier()).ToDedupIdentifier(); if (dedupId.AlgorithmId == Hashing.ChunkDedupIdentifier.ChunkAlgorithmId) { pinResult = await TryPinChunkAsync(context, dedupId); } else { pinResult = await TryPinNodeAsync(context, dedupId); } if (pinResult.Succeeded) { _counters[BackingContentStore.SessionCounters.PinSatisfiedFromRemote].Increment(); BackingContentStoreExpiryCache.Instance.AddExpiry(contentHash, EndDateTime); } return(pinResult); } catch (Exception ex) { return(new PinResult(ex)); } }
/// <inheritdoc /> protected override async Task <PinResult> PinCoreAsync( OperationContext context, ContentHash contentHash, UrgencyHint urgencyHint, Counter retryCounter) { if (!contentHash.HashType.IsValidDedup()) { return(new PinResult($"DedupStore client requires a HashType that supports dedup. Given hash type: {contentHash.HashType}.")); } var pinResult = CheckPinInMemory(contentHash); if (pinResult.Succeeded) { return(pinResult); } VstsBlobIdentifier blobId = ToVstsBlobIdentifier(contentHash.ToBlobIdentifier()); VstsDedupIdentifier dedupId = blobId.ToDedupIdentifier(); if (dedupId.AlgorithmId == Hashing.ChunkDedupIdentifier.ChunkAlgorithmId) { // No need to optimize since pinning a chunk is always a fast operation. return(await PinImplAsync(context, contentHash)); } // Since pinning the whole tree can be an expensive operation, we have optimized how we call it. Depending on the current // keepUntil of the root node, which is unexpensive to check, the operation will behave differently: // The pin operation will be ignored if it is greater than ignorePinThreshold, to reduce amount of calls // The pin operation will be inlined if it is lower than pinInlineThreshold, to make sure that we don't try to use // content that we pin in the background but has expired before we could complete the pin. // The pin operation will be done asynchronously and will return success otherwise. Most calls should follow this // behavior, to avoid waiting on a potentially long operation. We're confident returning a success because we // know that the content is there even though we still have to extend it's keepUntil var keepUntilResult = await CheckNodeKeepUntilAsync(context, dedupId); if (!keepUntilResult.Succeeded) { // Returned a service error. Fail fast. return(new PinResult(keepUntilResult)); } else if (!keepUntilResult.Value.HasValue) { // Content not found. return(new PinResult(PinResult.ResultCode.ContentNotFound)); } var timeLeft = keepUntilResult.Value.Value - DateTime.UtcNow; if (timeLeft > _ignorePinThreshold) { Tracer.Debug(context, $"Pin was skipped because keepUntil has remaining time [{timeLeft}] that is greater than ignorePinThreshold=[{_ignorePinThreshold}]"); _dedupCounters[Counters.PinIgnored].Increment(); return(PinResult.Success); } var pinTask = PinImplAsync(context, contentHash); if (timeLeft < _pinInlineThreshold) { Tracer.Debug(context, $"Pin inlined because keepUntil has remaining time [{timeLeft}] that is less than pinInlineThreshold=[{_pinInlineThreshold}]"); _dedupCounters[Counters.PinInlined].Increment(); return(await pinTask); } pinTask.FireAndForget(context); return(PinResult.Success); }