private async Task <IEnumerable <Task <Indexed <PinResult> > > > UpdateBlobStoreAsync(Context context, IReadOnlyList <ContentHash> contentHashes, DateTime endDateTime, CancellationToken cts) { // Convert missing content hashes to blob Ids var blobIds = contentHashes.Select(c => ToVstsBlobIdentifier(c.ToBlobIdentifier())).ToList(); // Call TryReference on the blob ids var references = blobIds.Distinct().ToDictionary( blobIdentifier => blobIdentifier, _ => (IEnumerable <BlobReference>) new[] { new BlobReference(endDateTime) }); // TODO: In groups of 1000 (bug 1365340) var referenceResults = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync( context, "UpdateBlobStore", innerCts => BlobStoreHttpClient.TryReferenceAsync(references, cancellationToken: innerCts), cts).ConfigureAwait(false); Tracer.RecordPinSatisfiedFromRemote(); // There's 1-1 mapping between given content hashes and blob ids var remoteResults = blobIds .Select((blobId, i) => { PinResult pinResult = referenceResults.ContainsKey(blobId) ? PinResult.ContentNotFound : PinResult.Success; if (pinResult.Succeeded) { BackingContentStoreExpiryCache.Instance.AddExpiry(contentHashes[i], endDateTime); } return(pinResult.WithIndex(i)); }); return(remoteResults.AsTasks()); }
private async Task <Stream> GetStreamInternalAsync(Context context, ContentHash contentHash, int?overrideStreamMinimumReadSizeInBytes, CancellationToken cts) { if (_downloadBlobsThroughBlobStore) { return(await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync( context, "GetStreamInternalThroughBlobStore", innerCts => BlobStoreHttpClient.GetBlobAsync(ToVstsBlobIdentifier(contentHash.ToBlobIdentifier()), cancellationToken: innerCts), cts).ConfigureAwait(false)); } else { PreauthenticatedUri uri; if (!DownloadUriCache.Instance.TryGetDownloadUri(contentHash, out uri)) { Tracer.RecordDownloadUriFetchedFromRemote(); BlobIdentifier blobId = contentHash.ToBlobIdentifier(); IDictionary <VstsBlobIdentifier, PreauthenticatedUri> mappings = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync( context, "GetStreamInternal", innerCts => BlobStoreHttpClient.GetDownloadUrisAsync( new[] { ToVstsBlobIdentifier(blobId) }, EdgeCache.NotAllowed, cancellationToken: innerCts), cts).ConfigureAwait(false); if (mappings == null || !mappings.TryGetValue(ToVstsBlobIdentifier(blobId), out uri)) { return(null); } DownloadUriCache.Instance.AddDownloadUri(contentHash, uri); } else { Tracer.RecordDownloadUriFetchedFromCache(); } return(await GetStreamThroughAzureBlobs( uri.NotNullUri, overrideStreamMinimumReadSizeInBytes, _parallelSegmentDownloadConfig.SegmentDownloadTimeout, cts).ConfigureAwait(false)); } }