/// <inheritdoc /> public Task <ContentHashListResponse> AddContentHashListAsync( string cacheNamespace, StrongFingerprint strongFingerprint, ContentHashListWithCacheMetadata contentHashList, bool forceUpdate) { var addContentHashListRequest = new AddContentHashListRequest(contentHashList); var contentHashListParameters = new { cacheNamespace, weakFingerprint = strongFingerprint.WeakFingerprint.ToHex(), selectorContentHash = strongFingerprint.Selector.ContentHash.ToHex(), selectorOutput = strongFingerprint.Selector.Output?.ToHex() ?? BuildCacheResourceIds.NoneSelectorOutput, }; var queryParameters = new Dictionary <string, string>(); if (forceUpdate) { queryParameters["forceUpdate"] = forceUpdate.ToString(); } return(PostAsync <AddContentHashListRequest, ContentHashListResponse>( addContentHashListRequest, BuildCacheResourceIds.ContentHashListResourceId, contentHashListParameters, queryParameters: queryParameters)); }
private void SealIfNecessaryAfterUnbackedAddOrGet( Context context, StrongFingerprint strongFingerprint, ContentHashListWithDeterminism addedValue, ContentHashListWithCacheMetadata cacheMetadata) { if (cacheMetadata == null) { // Our unbacked Add won the race, so the value is implicitly unbacked in VSTS // Seal the added value SealInTheBackground(context, strongFingerprint, addedValue); } else if (cacheMetadata.GetEffectiveExpirationTimeUtc() == null) { // Value is explicitly unbacked in VSTS if (cacheMetadata.ContentHashListWithDeterminism.ContentHashList != null) { // Our Add lost the race, so seal the existing value SealInTheBackground(context, strongFingerprint, cacheMetadata.ContentHashListWithDeterminism); } else { // Our Add won the race, so seal the added value var valueToSeal = new ContentHashListWithDeterminism( addedValue.ContentHashList, cacheMetadata.ContentHashListWithDeterminism.Determinism); SealInTheBackground( context, strongFingerprint, valueToSeal); } } }
/// <inheritdoc /> public async Task <ObjectResult <ContentHashListWithCacheMetadata> > AddContentHashListAsync( Context context, string cacheNamespace, StrongFingerprint strongFingerprint, ContentHashListWithCacheMetadata valueToAdd, bool forceUpdate) { try { ContentHashListResponse addResult = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync( context, "AddContentHashList", innerCts => _buildCacheHttpClient.AddContentHashListAsync( cacheNamespace, strongFingerprint, valueToAdd, forceUpdate), CancellationToken.None).ConfigureAwait(false); // The return value is null if the server fails adding content hash list to the backing store. // See BuildCacheService.AddContentHashListAsync for more details about the implementation invariants/guarantees. if (addResult != null) { DownloadUriCache.Instance.BulkAddDownloadUris(addResult.BlobDownloadUris); } // add succeeded but returned an empty contenthashlistwith cache metadata. correct this. if (addResult?.ContentHashListWithCacheMetadata == null) { return (new ObjectResult <ContentHashListWithCacheMetadata>( new ContentHashListWithCacheMetadata( new ContentHashListWithDeterminism(null, valueToAdd.ContentHashListWithDeterminism.Determinism), valueToAdd.GetRawExpirationTimeUtc(), valueToAdd.ContentGuarantee, valueToAdd.HashOfExistingContentHashList))); } else if (addResult.ContentHashListWithCacheMetadata.ContentHashListWithDeterminism.ContentHashList != null && addResult.ContentHashListWithCacheMetadata.HashOfExistingContentHashList == null) { return(new ObjectResult <ContentHashListWithCacheMetadata>( new ContentHashListWithCacheMetadata( addResult.ContentHashListWithCacheMetadata.ContentHashListWithDeterminism, addResult.ContentHashListWithCacheMetadata.GetRawExpirationTimeUtc(), addResult.ContentHashListWithCacheMetadata.ContentGuarantee, addResult.ContentHashListWithCacheMetadata.ContentHashListWithDeterminism.ContentHashList.GetHashOfHashes()))); } else { return(new ObjectResult <ContentHashListWithCacheMetadata>(addResult.ContentHashListWithCacheMetadata)); } } catch (Exception ex) { return(new ObjectResult <ContentHashListWithCacheMetadata>(ex)); } }
private async Task <bool> CheckNeedToUpdateExistingValueAsync( Context context, ContentHashListWithCacheMetadata cacheMetadata, ContentHashList contentHashListToReturn, CancellationToken cts, UrgencyHint urgencyHint) { return(cacheMetadata != null && cacheMetadata.GetEffectiveExpirationTimeUtc() == null && contentHashListToReturn != null && (!await EnsureContentIsAvailableAsync(context, contentHashListToReturn.Hashes, cts, urgencyHint).ConfigureAwait(false))); }
/// <summary> /// Checks for inconsistencies in the metadata returned by the service, returning an appropriate error message (null if none). /// </summary> protected static string CheckForResponseInconsistency(ContentHashListWithCacheMetadata cacheMetadata) { if (cacheMetadata != null) { if (cacheMetadata.GetEffectiveExpirationTimeUtc() == null && cacheMetadata.ContentGuarantee != ContentAvailabilityGuarantee.NoContentBackedByCache) { return ("Inconsistent BuildCache service response. Null ContentHashListExpirationUtc should be iff ContentAvailabilityGuarantee.NoContentBackedByCache."); } } return(null); }
/// <inheritdoc /> public async Task <ObjectResult <ContentHashListWithCacheMetadata> > AddContentHashListAsync( Context context, string cacheNamespace, StrongFingerprint strongFingerprint, ContentHashListWithCacheMetadata valueToAdd) { try { ContentHashListResponse addResult = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync( context, "AddContentHashList", innerCts => _buildCacheHttpClient.AddContentHashListAsync( cacheNamespace, strongFingerprint, valueToAdd), CancellationToken.None).ConfigureAwait(false); DownloadUriCache.Instance.BulkAddDownloadUris(addResult.BlobDownloadUris); // add succeeded but returned an empty contenthashlistwith cache metadata. correct this. if (addResult.ContentHashListWithCacheMetadata == null) { return (new ObjectResult <ContentHashListWithCacheMetadata>( new ContentHashListWithCacheMetadata( new ContentHashListWithDeterminism(null, valueToAdd.ContentHashListWithDeterminism.Determinism), valueToAdd.GetEffectiveExpirationTimeUtc(), valueToAdd.ContentGuarantee, valueToAdd.HashOfExistingContentHashList))); } else if (addResult.ContentHashListWithCacheMetadata.ContentHashListWithDeterminism.ContentHashList != null && addResult.ContentHashListWithCacheMetadata.HashOfExistingContentHashList == null) { return(new ObjectResult <ContentHashListWithCacheMetadata>( new ContentHashListWithCacheMetadata( addResult.ContentHashListWithCacheMetadata.ContentHashListWithDeterminism, addResult.ContentHashListWithCacheMetadata.GetEffectiveExpirationTimeUtc(), addResult.ContentHashListWithCacheMetadata.ContentGuarantee, addResult.ContentHashListWithCacheMetadata.ContentHashListWithDeterminism.ContentHashList.GetHashOfHashes()))); } else { return(new ObjectResult <ContentHashListWithCacheMetadata>(addResult.ContentHashListWithCacheMetadata)); } } catch (Exception ex) { return(new ObjectResult <ContentHashListWithCacheMetadata>(ex)); } }
/// <inheritdoc /> public Task <GetContentHashListResult> GetContentHashListAsync( Context context, StrongFingerprint strongFingerprint, CancellationToken cts, UrgencyHint urgencyHint) { return(GetContentHashListCall.RunAsync(Tracer.MemoizationStoreTracer, context, strongFingerprint, async() => { // Check for pre-fetched data ContentHashListWithDeterminism contentHashListWithDeterminism; if (ContentHashListWithDeterminismCache.Instance.TryGetValue( CacheNamespace, strongFingerprint, out contentHashListWithDeterminism)) { Tracer.RecordUseOfPrefetchedContentHashList(); FingerprintTracker.Track( strongFingerprint, contentHashListWithDeterminism.Determinism.ExpirationUtc); return new GetContentHashListResult(contentHashListWithDeterminism); } // No pre-fetched data. Need to query the server. ObjectResult <ContentHashListWithCacheMetadata> responseObject = await ContentHashListAdapter.GetContentHashListAsync(context, CacheNamespace, strongFingerprint).ConfigureAwait(false); if (!responseObject.Succeeded) { return new GetContentHashListResult(responseObject); } ContentHashListWithCacheMetadata response = responseObject.Data; if (response.ContentHashListWithDeterminism.ContentHashList == null) { // Miss return new GetContentHashListResult(new ContentHashListWithDeterminism(null, CacheDeterminism.None)); } GetContentHashListResult unpackResult = UnpackContentHashListWithDeterminismAfterGet(response, CacheId); if (!unpackResult.Succeeded) { return unpackResult; } SealIfNecessaryAfterGet(context, strongFingerprint, response); FingerprintTracker.Track(strongFingerprint, response.GetRawExpirationTimeUtc()); return new GetContentHashListResult(unpackResult.ContentHashListWithDeterminism); })); }
/// <inheritdoc /> protected async Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync( Context context, StrongFingerprint strongFingerprint, ContentHashListWithDeterminism contentHashListWithDeterminism, ContentAvailabilityGuarantee guarantee) { try { DateTime expirationUtc = FingerprintTracker.GenerateNewExpiration(); var valueToAdd = new ContentHashListWithCacheMetadata( contentHashListWithDeterminism, expirationUtc, guarantee); Tracer.Debug( context, $"Adding contentHashList=[{valueToAdd.ContentHashListWithDeterminism.ContentHashList}] determinism=[{valueToAdd.ContentHashListWithDeterminism.Determinism}] to VSTS with contentAvailabilityGuarantee=[{valueToAdd.ContentGuarantee}] and expirationUtc=[{expirationUtc}]"); var contentHashListResponseObject = await ContentHashListAdapter.AddContentHashListAsync(context, CacheNamespace, strongFingerprint, valueToAdd).ConfigureAwait(false); if (!contentHashListResponseObject.Succeeded) { return(new AddOrGetContentHashListResult(contentHashListResponseObject)); } var contentHashListResponse = contentHashListResponseObject.Data; var inconsistencyErrorMessage = CheckForResponseInconsistency(contentHashListResponse); if (inconsistencyErrorMessage != null) { return(new AddOrGetContentHashListResult(inconsistencyErrorMessage)); } ContentHashList contentHashListToReturn = UnpackContentHashListAfterAdd( contentHashListWithDeterminism.ContentHashList, contentHashListResponse); CacheDeterminism determinismToReturn = UnpackDeterminism(contentHashListResponse, CacheId); if (guarantee == ContentAvailabilityGuarantee.AllContentBackedByCache && !determinismToReturn.IsDeterministic) { return(new AddOrGetContentHashListResult( "Inconsistent BuildCache service response. Unbacked values should never override backed values.")); } FingerprintTracker.Track(strongFingerprint, contentHashListResponse.GetRawExpirationTimeUtc()); return(new AddOrGetContentHashListResult(new ContentHashListWithDeterminism(contentHashListToReturn, determinismToReturn))); } catch (Exception e) { return(new AddOrGetContentHashListResult(e)); } }
/// <summary> /// Determine the Determinism to return. /// </summary> protected static CacheDeterminism UnpackDeterminism(ContentHashListWithCacheMetadata cacheMetadata, Guid cacheId) { Contract.Assert(cacheMetadata != null); if (cacheMetadata.ContentHashListWithDeterminism.Determinism.Guid == CacheDeterminism.Tool.Guid) { // Value is Tool-deterministic return(CacheDeterminism.Tool); } var expirationUtc = cacheMetadata.GetEffectiveExpirationTimeUtc(); return(expirationUtc == null ? CacheDeterminism.None // Value is unbacked in VSTS : CacheDeterminism.ViaCache(cacheId, expirationUtc.Value)); // Value is backed in VSTS }
/// <summary> /// Determine the ContentHashList to return based on the added value and the service response. /// </summary> protected static ContentHashList UnpackContentHashListAfterAdd( ContentHashList addedContentHashList, ContentHashListWithCacheMetadata cacheMetadata) { Contract.Assert(cacheMetadata != null); if (cacheMetadata.ContentHashListWithDeterminism.ContentHashList != null && !addedContentHashList.Equals( cacheMetadata.ContentHashListWithDeterminism.ContentHashList)) { // The service returned a ContentHashList different from the one we tried to add, so we'll return that. return(cacheMetadata.ContentHashListWithDeterminism.ContentHashList); } // The added value was accepted, so we return null. return(null); }
private async Task <ObjectResult <ContentHashListWithCacheMetadata> > UnpackBlobContentHashListAsync(Context context, BlobContentHashListWithCacheMetadata blobCacheMetadata) { Contract.Assert(blobCacheMetadata != null); if (blobCacheMetadata.ContentHashListWithDeterminism.BlobIdentifier == null) { return(new ObjectResult <ContentHashListWithCacheMetadata>( new ContentHashListWithCacheMetadata( new ContentHashListWithDeterminism(null, blobCacheMetadata.Determinism), blobCacheMetadata.GetRawExpirationTimeUtc(), blobCacheMetadata.ContentGuarantee, blobCacheMetadata.HashOfExistingContentHashList))); } BlobIdentifier blobId = blobCacheMetadata.ContentHashListWithDeterminism.BlobIdentifier; Func <ContentHash, CancellationToken, Task <ObjectResult <Stream> > > openStreamFunc = async(hash, cts) => { OpenStreamResult openStreamResult = await _blobContentSession.OpenStreamAsync(context, hash, cts); if (openStreamResult.Succeeded) { return(new ObjectResult <Stream>(openStreamResult.Stream)); } return(new ObjectResult <Stream>(openStreamResult)); }; StructResult <ContentHashListWithDeterminism> contentHashListResult = await BlobContentHashListExtensions.UnpackFromBlob( openStreamFunc, blobId); if (contentHashListResult.Succeeded) { var contentHashListWithCacheMetadata = new ContentHashListWithCacheMetadata( contentHashListResult.Data, blobCacheMetadata.GetRawExpirationTimeUtc(), blobCacheMetadata.ContentGuarantee, blobCacheMetadata.HashOfExistingContentHashList); return(new ObjectResult <ContentHashListWithCacheMetadata>(contentHashListWithCacheMetadata)); } else { return(new ObjectResult <ContentHashListWithCacheMetadata>(contentHashListResult)); } }
/// <inheritdoc /> public Task <ContentHashListResponse> AddContentHashListAsync( string cacheNamespace, StrongFingerprint strongFingerprint, ContentHashListWithCacheMetadata contentHashList) { var addContentHashListRequest = new AddContentHashListRequest(contentHashList); var contentHashListParameters = new { cacheNamespace, weakFingerprint = strongFingerprint.WeakFingerprint.ToHex(), selectorContentHash = strongFingerprint.Selector.ContentHash.ToHex(), selectorOutput = strongFingerprint.Selector.Output?.ToHex() ?? BuildCacheResourceIds.NoneSelectorOutput }; return(PostAsync <AddContentHashListRequest, ContentHashListResponse>( addContentHashListRequest, BuildCacheResourceIds.ContentHashListResourceId, contentHashListParameters)); }
private static GetContentHashListResult UnpackContentHashListWithDeterminismAfterGet( ContentHashListWithCacheMetadata cacheMetadata, Guid cacheId) { var inconsistencyErrorMessage = CheckForResponseInconsistency(cacheMetadata); if (inconsistencyErrorMessage != null) { return(new GetContentHashListResult(inconsistencyErrorMessage)); } if (cacheMetadata?.ContentHashListWithDeterminism.ContentHashList == null) { // Miss return(new GetContentHashListResult(new ContentHashListWithDeterminism(null, CacheDeterminism.None))); } ContentHashList contentHashList = cacheMetadata.ContentHashListWithDeterminism.ContentHashList; CacheDeterminism determinism = UnpackDeterminism(cacheMetadata, cacheId); return(new GetContentHashListResult(new ContentHashListWithDeterminism(contentHashList, determinism))); }
/// <summary> /// Initializes a new instance of the <see cref="SelectorAndContentHashListWithCacheMetadata"/> struct. /// </summary> public SelectorAndContentHashListWithCacheMetadata(Selector selector, ContentHashListWithCacheMetadata contentHashList) { Selector = selector; ContentHashList = contentHashList; }
/// <inheritdoc /> public async Task <ObjectResult <ContentHashListWithCacheMetadata> > AddContentHashListAsync( Context context, string cacheNamespace, StrongFingerprint strongFingerprint, ContentHashListWithCacheMetadata valueToAdd) { try { Func <System.IO.Stream, System.Threading.CancellationToken, Task <StructResult <ContentHash> > > putStreamFunc = async(stream, cts) => { PutResult putResult = await _blobContentSession.PutStreamAsync(context, HashType.Vso0, stream, cts); if (putResult.Succeeded) { return(new StructResult <ContentHash>(putResult.ContentHash)); } return(new StructResult <ContentHash>(putResult)); }; StructResult <ContentHash> blobIdOfContentHashListResult = await BlobContentHashListExtensions.PackInBlob( putStreamFunc, valueToAdd.ContentHashListWithDeterminism); if (!blobIdOfContentHashListResult.Succeeded) { return(new ObjectResult <ContentHashListWithCacheMetadata>(blobIdOfContentHashListResult)); } var blobContentHashListWithDeterminism = new BlobContentHashListWithDeterminism( valueToAdd.ContentHashListWithDeterminism.Determinism.EffectiveGuid, BlobIdentifierToContentHashExtensions.ToBlobIdentifier(blobIdOfContentHashListResult.Data)); var blobContentHashListWithCacheMetadata = new BlobContentHashListWithCacheMetadata( blobContentHashListWithDeterminism, valueToAdd.GetRawExpirationTimeUtc(), valueToAdd.ContentGuarantee, valueToAdd.HashOfExistingContentHashList); BlobContentHashListResponse addResult = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync( context, "AddContentHashList", innerCts => _buildCacheHttpClient.AddContentHashListAsync( cacheNamespace, strongFingerprint, blobContentHashListWithCacheMetadata), CancellationToken.None).ConfigureAwait(false); DownloadUriCache.Instance.BulkAddDownloadUris(addResult.BlobDownloadUris); // add succeeded but returned an empty contenthashlistwith cache metadata. correct this. if (addResult.ContentHashListWithCacheMetadata == null) { return (new ObjectResult <ContentHashListWithCacheMetadata>( new ContentHashListWithCacheMetadata( new ContentHashListWithDeterminism(null, blobContentHashListWithCacheMetadata.Determinism), blobContentHashListWithCacheMetadata.GetRawExpirationTimeUtc(), blobContentHashListWithCacheMetadata.ContentGuarantee))); } else { return(await UnpackBlobContentHashListAsync(context, addResult.ContentHashListWithCacheMetadata)); } } catch (Exception ex) { return(new ObjectResult <ContentHashListWithCacheMetadata>(ex)); } }
private void SealIfNecessaryAfterGet(Context context, StrongFingerprint strongFingerprint, ContentHashListWithCacheMetadata cacheMetadata) { if (WriteThroughContentSession == null) { return; } if (cacheMetadata != null && cacheMetadata.GetEffectiveExpirationTimeUtc() == null) { // Value is unbacked in VSTS SealInTheBackground(context, strongFingerprint, cacheMetadata.ContentHashListWithDeterminism); } }
/// <inheritdoc /> public Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync( Context context, StrongFingerprint strongFingerprint, ContentHashListWithDeterminism contentHashListWithDeterminism, CancellationToken cts, UrgencyHint urgencyHint) { return(new OperationContext(context, cts).PerformOperationAsync( Tracer.MemoizationStoreTracer, async() => { // TODO: Split this out into separate implementations for WriteThrough vs. WriteBehind (bug 1365340) if (WriteThroughContentSession == null) { ContentAvailabilityGuarantee guarantee = ManuallyExtendContentLifetime ? ContentAvailabilityGuarantee.NoContentBackedByCache : ContentAvailabilityGuarantee.AllContentBackedByCache; return await AddOrGetContentHashListAsync( context, strongFingerprint, contentHashListWithDeterminism, guarantee).ConfigureAwait(false); } // Ensure that the content exists somewhere before trying to add if (!await EnsureContentIsAvailableAsync( context, contentHashListWithDeterminism.ContentHashList.Hashes, cts, urgencyHint).ConfigureAwait(false)) { return new AddOrGetContentHashListResult( "Referenced content must exist in the cache before a new content hash list is added."); } DateTime expirationUtc = FingerprintTracker.GenerateNewExpiration(); var valueToAdd = new ContentHashListWithCacheMetadata( contentHashListWithDeterminism, expirationUtc, ContentAvailabilityGuarantee.NoContentBackedByCache); DateTime?rawExpiration = null; const int addLimit = 3; for (int addAttempts = 0; addAttempts < addLimit; addAttempts++) { var debugString = $"Adding contentHashList=[{valueToAdd.ContentHashListWithDeterminism.ContentHashList}] " + $"determinism=[{valueToAdd.ContentHashListWithDeterminism.Determinism}] to VSTS with " + $"contentAvailabilityGuarantee=[{valueToAdd.ContentGuarantee}], expirationUtc=[{expirationUtc}], forceUpdate=[{ForceUpdateOnAddContentHashList}]"; Tracer.Debug(context, debugString); Result <ContentHashListWithCacheMetadata> responseObject = await ContentHashListAdapter.AddContentHashListAsync( context, CacheNamespace, strongFingerprint, valueToAdd, forceUpdate: ForceUpdateOnAddContentHashList).ConfigureAwait(false); if (!responseObject.Succeeded) { return new AddOrGetContentHashListResult(responseObject); } ContentHashListWithCacheMetadata response = responseObject.Value; var inconsistencyErrorMessage = CheckForResponseInconsistency(response); if (inconsistencyErrorMessage != null) { return new AddOrGetContentHashListResult(inconsistencyErrorMessage); } rawExpiration = response.GetRawExpirationTimeUtc(); ContentHashList contentHashListToReturn = UnpackContentHashListAfterAdd(contentHashListWithDeterminism.ContentHashList, response); CacheDeterminism determinismToReturn = UnpackDeterminism(response, CacheId); bool needToUpdateExistingValue = await CheckNeedToUpdateExistingValueAsync( context, response, contentHashListToReturn, cts, urgencyHint).ConfigureAwait(false); if (!needToUpdateExistingValue) { SealIfNecessaryAfterUnbackedAddOrGet(context, strongFingerprint, contentHashListWithDeterminism, response); await TrackFingerprintAsync(context, strongFingerprint, rawExpiration).ConfigureAwait(false); return new AddOrGetContentHashListResult( new ContentHashListWithDeterminism(contentHashListToReturn, determinismToReturn)); } var hashOfExistingContentHashList = response.HashOfExistingContentHashList; Tracer.Debug(context, $"Attempting to replace unbacked value with hash {hashOfExistingContentHashList.ToHex()}"); valueToAdd = new ContentHashListWithCacheMetadata( contentHashListWithDeterminism, expirationUtc, ContentAvailabilityGuarantee.NoContentBackedByCache, hashOfExistingContentHashList ); } Tracer.Warning( context, $"Lost the AddOrUpdate race {addLimit} times against unbacked values. Returning as though the add succeeded for now."); await TrackFingerprintAsync(context, strongFingerprint, rawExpiration).ConfigureAwait(false); return new AddOrGetContentHashListResult(new ContentHashListWithDeterminism(null, CacheDeterminism.None)); }, traceOperationStarted: true, extraStartMessage: $"StrongFingerprint=({strongFingerprint}), ForceUpdate=({ForceUpdateOnAddContentHashList}) {contentHashListWithDeterminism.ToTraceString()}", extraEndMessage: _ => $"StrongFingerprint=({strongFingerprint}), ForceUpdate=({ForceUpdateOnAddContentHashList}) {contentHashListWithDeterminism.ToTraceString()}")); }