Example #1
0
        /// <inheritdoc />
        public Task <ContentHashListResponse> AddContentHashListAsync(
            string cacheNamespace,
            StrongFingerprint strongFingerprint,
            ContentHashListWithCacheMetadata contentHashList,
            bool forceUpdate)
        {
            var addContentHashListRequest = new AddContentHashListRequest(contentHashList);

            var contentHashListParameters = new
            {
                cacheNamespace,
                weakFingerprint     = strongFingerprint.WeakFingerprint.ToHex(),
                selectorContentHash = strongFingerprint.Selector.ContentHash.ToHex(),
                selectorOutput      = strongFingerprint.Selector.Output?.ToHex() ?? BuildCacheResourceIds.NoneSelectorOutput,
            };

            var queryParameters = new Dictionary <string, string>();

            if (forceUpdate)
            {
                queryParameters["forceUpdate"] = forceUpdate.ToString();
            }

            return(PostAsync <AddContentHashListRequest, ContentHashListResponse>(
                       addContentHashListRequest,
                       BuildCacheResourceIds.ContentHashListResourceId,
                       contentHashListParameters,
                       queryParameters: queryParameters));
        }
Example #2
0
 private void SealIfNecessaryAfterUnbackedAddOrGet(
     Context context,
     StrongFingerprint strongFingerprint,
     ContentHashListWithDeterminism addedValue,
     ContentHashListWithCacheMetadata cacheMetadata)
 {
     if (cacheMetadata == null)
     {
         // Our unbacked Add won the race, so the value is implicitly unbacked in VSTS
         // Seal the added value
         SealInTheBackground(context, strongFingerprint, addedValue);
     }
     else if (cacheMetadata.GetEffectiveExpirationTimeUtc() == null)
     {
         // Value is explicitly unbacked in VSTS
         if (cacheMetadata.ContentHashListWithDeterminism.ContentHashList != null)
         {
             // Our Add lost the race, so seal the existing value
             SealInTheBackground(context, strongFingerprint, cacheMetadata.ContentHashListWithDeterminism);
         }
         else
         {
             // Our Add won the race, so seal the added value
             var valueToSeal = new ContentHashListWithDeterminism(
                 addedValue.ContentHashList, cacheMetadata.ContentHashListWithDeterminism.Determinism);
             SealInTheBackground(
                 context,
                 strongFingerprint,
                 valueToSeal);
         }
     }
 }
        /// <inheritdoc />
        public async Task <ObjectResult <ContentHashListWithCacheMetadata> > AddContentHashListAsync(
            Context context,
            string cacheNamespace,
            StrongFingerprint strongFingerprint,
            ContentHashListWithCacheMetadata valueToAdd,
            bool forceUpdate)
        {
            try
            {
                ContentHashListResponse addResult = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync(
                    context,
                    "AddContentHashList",
                    innerCts => _buildCacheHttpClient.AddContentHashListAsync(
                        cacheNamespace,
                        strongFingerprint,
                        valueToAdd,
                        forceUpdate), CancellationToken.None).ConfigureAwait(false);

                // The return value is null if the server fails adding content hash list to the backing store.
                // See BuildCacheService.AddContentHashListAsync for more details about the implementation invariants/guarantees.
                if (addResult != null)
                {
                    DownloadUriCache.Instance.BulkAddDownloadUris(addResult.BlobDownloadUris);
                }

                // add succeeded but returned an empty contenthashlistwith cache metadata. correct this.
                if (addResult?.ContentHashListWithCacheMetadata == null)
                {
                    return
                        (new ObjectResult <ContentHashListWithCacheMetadata>(
                             new ContentHashListWithCacheMetadata(
                                 new ContentHashListWithDeterminism(null, valueToAdd.ContentHashListWithDeterminism.Determinism),
                                 valueToAdd.GetRawExpirationTimeUtc(),
                                 valueToAdd.ContentGuarantee,
                                 valueToAdd.HashOfExistingContentHashList)));
                }
                else if (addResult.ContentHashListWithCacheMetadata.ContentHashListWithDeterminism.ContentHashList != null &&
                         addResult.ContentHashListWithCacheMetadata.HashOfExistingContentHashList == null)
                {
                    return(new ObjectResult <ContentHashListWithCacheMetadata>(
                               new ContentHashListWithCacheMetadata(
                                   addResult.ContentHashListWithCacheMetadata.ContentHashListWithDeterminism,
                                   addResult.ContentHashListWithCacheMetadata.GetRawExpirationTimeUtc(),
                                   addResult.ContentHashListWithCacheMetadata.ContentGuarantee,
                                   addResult.ContentHashListWithCacheMetadata.ContentHashListWithDeterminism.ContentHashList.GetHashOfHashes())));
                }
                else
                {
                    return(new ObjectResult <ContentHashListWithCacheMetadata>(addResult.ContentHashListWithCacheMetadata));
                }
            }
            catch (Exception ex)
            {
                return(new ObjectResult <ContentHashListWithCacheMetadata>(ex));
            }
        }
Example #4
0
 private async Task <bool> CheckNeedToUpdateExistingValueAsync(
     Context context,
     ContentHashListWithCacheMetadata cacheMetadata,
     ContentHashList contentHashListToReturn,
     CancellationToken cts,
     UrgencyHint urgencyHint)
 {
     return(cacheMetadata != null &&
            cacheMetadata.GetEffectiveExpirationTimeUtc() == null &&
            contentHashListToReturn != null &&
            (!await EnsureContentIsAvailableAsync(context, contentHashListToReturn.Hashes, cts, urgencyHint).ConfigureAwait(false)));
 }
        /// <summary>
        ///     Checks for inconsistencies in the metadata returned by the service, returning an appropriate error message (null if none).
        /// </summary>
        protected static string CheckForResponseInconsistency(ContentHashListWithCacheMetadata cacheMetadata)
        {
            if (cacheMetadata != null)
            {
                if (cacheMetadata.GetEffectiveExpirationTimeUtc() == null &&
                    cacheMetadata.ContentGuarantee != ContentAvailabilityGuarantee.NoContentBackedByCache)
                {
                    return
                        ("Inconsistent BuildCache service response. Null ContentHashListExpirationUtc should be iff ContentAvailabilityGuarantee.NoContentBackedByCache.");
                }
            }

            return(null);
        }
        /// <inheritdoc />
        public async Task <ObjectResult <ContentHashListWithCacheMetadata> > AddContentHashListAsync(
            Context context,
            string cacheNamespace,
            StrongFingerprint strongFingerprint,
            ContentHashListWithCacheMetadata valueToAdd)
        {
            try
            {
                ContentHashListResponse addResult = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync(
                    context,
                    "AddContentHashList",
                    innerCts => _buildCacheHttpClient.AddContentHashListAsync(
                        cacheNamespace,
                        strongFingerprint,
                        valueToAdd), CancellationToken.None).ConfigureAwait(false);

                DownloadUriCache.Instance.BulkAddDownloadUris(addResult.BlobDownloadUris);

                // add succeeded but returned an empty contenthashlistwith cache metadata. correct this.
                if (addResult.ContentHashListWithCacheMetadata == null)
                {
                    return
                        (new ObjectResult <ContentHashListWithCacheMetadata>(
                             new ContentHashListWithCacheMetadata(
                                 new ContentHashListWithDeterminism(null, valueToAdd.ContentHashListWithDeterminism.Determinism),
                                 valueToAdd.GetEffectiveExpirationTimeUtc(),
                                 valueToAdd.ContentGuarantee,
                                 valueToAdd.HashOfExistingContentHashList)));
                }
                else if (addResult.ContentHashListWithCacheMetadata.ContentHashListWithDeterminism.ContentHashList != null &&
                         addResult.ContentHashListWithCacheMetadata.HashOfExistingContentHashList == null)
                {
                    return(new ObjectResult <ContentHashListWithCacheMetadata>(
                               new ContentHashListWithCacheMetadata(
                                   addResult.ContentHashListWithCacheMetadata.ContentHashListWithDeterminism,
                                   addResult.ContentHashListWithCacheMetadata.GetEffectiveExpirationTimeUtc(),
                                   addResult.ContentHashListWithCacheMetadata.ContentGuarantee,
                                   addResult.ContentHashListWithCacheMetadata.ContentHashListWithDeterminism.ContentHashList.GetHashOfHashes())));
                }
                else
                {
                    return(new ObjectResult <ContentHashListWithCacheMetadata>(addResult.ContentHashListWithCacheMetadata));
                }
            }
            catch (Exception ex)
            {
                return(new ObjectResult <ContentHashListWithCacheMetadata>(ex));
            }
        }
        /// <inheritdoc />
        public Task <GetContentHashListResult> GetContentHashListAsync(
            Context context,
            StrongFingerprint strongFingerprint,
            CancellationToken cts,
            UrgencyHint urgencyHint)
        {
            return(GetContentHashListCall.RunAsync(Tracer.MemoizationStoreTracer, context, strongFingerprint, async() =>
            {
                // Check for pre-fetched data
                ContentHashListWithDeterminism contentHashListWithDeterminism;

                if (ContentHashListWithDeterminismCache.Instance.TryGetValue(
                        CacheNamespace, strongFingerprint, out contentHashListWithDeterminism))
                {
                    Tracer.RecordUseOfPrefetchedContentHashList();
                    FingerprintTracker.Track(
                        strongFingerprint,
                        contentHashListWithDeterminism.Determinism.ExpirationUtc);
                    return new GetContentHashListResult(contentHashListWithDeterminism);
                }

                // No pre-fetched data. Need to query the server.
                ObjectResult <ContentHashListWithCacheMetadata> responseObject =
                    await ContentHashListAdapter.GetContentHashListAsync(context, CacheNamespace, strongFingerprint).ConfigureAwait(false);

                if (!responseObject.Succeeded)
                {
                    return new GetContentHashListResult(responseObject);
                }

                ContentHashListWithCacheMetadata response = responseObject.Data;
                if (response.ContentHashListWithDeterminism.ContentHashList == null)
                {
                    // Miss
                    return new GetContentHashListResult(new ContentHashListWithDeterminism(null, CacheDeterminism.None));
                }

                GetContentHashListResult unpackResult = UnpackContentHashListWithDeterminismAfterGet(response, CacheId);
                if (!unpackResult.Succeeded)
                {
                    return unpackResult;
                }

                SealIfNecessaryAfterGet(context, strongFingerprint, response);
                FingerprintTracker.Track(strongFingerprint, response.GetRawExpirationTimeUtc());
                return new GetContentHashListResult(unpackResult.ContentHashListWithDeterminism);
            }));
        }
        /// <inheritdoc />
        protected async Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync(
            Context context,
            StrongFingerprint strongFingerprint,
            ContentHashListWithDeterminism contentHashListWithDeterminism,
            ContentAvailabilityGuarantee guarantee)
        {
            try
            {
                DateTime expirationUtc = FingerprintTracker.GenerateNewExpiration();
                var      valueToAdd    = new ContentHashListWithCacheMetadata(
                    contentHashListWithDeterminism, expirationUtc, guarantee);

                Tracer.Debug(
                    context,
                    $"Adding contentHashList=[{valueToAdd.ContentHashListWithDeterminism.ContentHashList}] determinism=[{valueToAdd.ContentHashListWithDeterminism.Determinism}] to VSTS with contentAvailabilityGuarantee=[{valueToAdd.ContentGuarantee}] and expirationUtc=[{expirationUtc}]");

                var contentHashListResponseObject = await ContentHashListAdapter.AddContentHashListAsync(context, CacheNamespace, strongFingerprint, valueToAdd).ConfigureAwait(false);

                if (!contentHashListResponseObject.Succeeded)
                {
                    return(new AddOrGetContentHashListResult(contentHashListResponseObject));
                }

                var contentHashListResponse   = contentHashListResponseObject.Data;
                var inconsistencyErrorMessage = CheckForResponseInconsistency(contentHashListResponse);
                if (inconsistencyErrorMessage != null)
                {
                    return(new AddOrGetContentHashListResult(inconsistencyErrorMessage));
                }

                ContentHashList contentHashListToReturn = UnpackContentHashListAfterAdd(
                    contentHashListWithDeterminism.ContentHashList, contentHashListResponse);

                CacheDeterminism determinismToReturn = UnpackDeterminism(contentHashListResponse, CacheId);
                if (guarantee == ContentAvailabilityGuarantee.AllContentBackedByCache && !determinismToReturn.IsDeterministic)
                {
                    return(new AddOrGetContentHashListResult(
                               "Inconsistent BuildCache service response. Unbacked values should never override backed values."));
                }

                FingerprintTracker.Track(strongFingerprint, contentHashListResponse.GetRawExpirationTimeUtc());
                return(new AddOrGetContentHashListResult(new ContentHashListWithDeterminism(contentHashListToReturn, determinismToReturn)));
            }
            catch (Exception e)
            {
                return(new AddOrGetContentHashListResult(e));
            }
        }
        /// <summary>
        ///     Determine the Determinism to return.
        /// </summary>
        protected static CacheDeterminism UnpackDeterminism(ContentHashListWithCacheMetadata cacheMetadata, Guid cacheId)
        {
            Contract.Assert(cacheMetadata != null);

            if (cacheMetadata.ContentHashListWithDeterminism.Determinism.Guid == CacheDeterminism.Tool.Guid)
            {
                // Value is Tool-deterministic
                return(CacheDeterminism.Tool);
            }

            var expirationUtc = cacheMetadata.GetEffectiveExpirationTimeUtc();

            return(expirationUtc == null
                ? CacheDeterminism.None                                     // Value is unbacked in VSTS
                : CacheDeterminism.ViaCache(cacheId, expirationUtc.Value)); // Value is backed in VSTS
        }
        /// <summary>
        ///     Determine the ContentHashList to return based on the added value and the service response.
        /// </summary>
        protected static ContentHashList UnpackContentHashListAfterAdd(
            ContentHashList addedContentHashList, ContentHashListWithCacheMetadata cacheMetadata)
        {
            Contract.Assert(cacheMetadata != null);

            if (cacheMetadata.ContentHashListWithDeterminism.ContentHashList != null &&
                !addedContentHashList.Equals(
                    cacheMetadata.ContentHashListWithDeterminism.ContentHashList))
            {
                // The service returned a ContentHashList different from the one we tried to add, so we'll return that.
                return(cacheMetadata.ContentHashListWithDeterminism.ContentHashList);
            }

            // The added value was accepted, so we return null.
            return(null);
        }
        private async Task <ObjectResult <ContentHashListWithCacheMetadata> > UnpackBlobContentHashListAsync(Context context, BlobContentHashListWithCacheMetadata blobCacheMetadata)
        {
            Contract.Assert(blobCacheMetadata != null);
            if (blobCacheMetadata.ContentHashListWithDeterminism.BlobIdentifier == null)
            {
                return(new ObjectResult <ContentHashListWithCacheMetadata>(
                           new ContentHashListWithCacheMetadata(
                               new ContentHashListWithDeterminism(null, blobCacheMetadata.Determinism),
                               blobCacheMetadata.GetRawExpirationTimeUtc(),
                               blobCacheMetadata.ContentGuarantee,
                               blobCacheMetadata.HashOfExistingContentHashList)));
            }

            BlobIdentifier blobId = blobCacheMetadata.ContentHashListWithDeterminism.BlobIdentifier;

            Func <ContentHash, CancellationToken, Task <ObjectResult <Stream> > > openStreamFunc = async(hash, cts) =>
            {
                OpenStreamResult openStreamResult = await _blobContentSession.OpenStreamAsync(context, hash, cts);

                if (openStreamResult.Succeeded)
                {
                    return(new ObjectResult <Stream>(openStreamResult.Stream));
                }

                return(new ObjectResult <Stream>(openStreamResult));
            };

            StructResult <ContentHashListWithDeterminism> contentHashListResult =
                await BlobContentHashListExtensions.UnpackFromBlob(
                    openStreamFunc,
                    blobId);

            if (contentHashListResult.Succeeded)
            {
                var contentHashListWithCacheMetadata = new ContentHashListWithCacheMetadata(
                    contentHashListResult.Data,
                    blobCacheMetadata.GetRawExpirationTimeUtc(),
                    blobCacheMetadata.ContentGuarantee,
                    blobCacheMetadata.HashOfExistingContentHashList);
                return(new ObjectResult <ContentHashListWithCacheMetadata>(contentHashListWithCacheMetadata));
            }
            else
            {
                return(new ObjectResult <ContentHashListWithCacheMetadata>(contentHashListResult));
            }
        }
Example #12
0
        /// <inheritdoc />
        public Task <ContentHashListResponse> AddContentHashListAsync(
            string cacheNamespace,
            StrongFingerprint strongFingerprint,
            ContentHashListWithCacheMetadata contentHashList)
        {
            var addContentHashListRequest = new AddContentHashListRequest(contentHashList);

            var contentHashListParameters = new
            {
                cacheNamespace,
                weakFingerprint     = strongFingerprint.WeakFingerprint.ToHex(),
                selectorContentHash = strongFingerprint.Selector.ContentHash.ToHex(),
                selectorOutput      = strongFingerprint.Selector.Output?.ToHex() ?? BuildCacheResourceIds.NoneSelectorOutput
            };

            return(PostAsync <AddContentHashListRequest, ContentHashListResponse>(
                       addContentHashListRequest,
                       BuildCacheResourceIds.ContentHashListResourceId,
                       contentHashListParameters));
        }
        private static GetContentHashListResult UnpackContentHashListWithDeterminismAfterGet(
            ContentHashListWithCacheMetadata cacheMetadata, Guid cacheId)
        {
            var inconsistencyErrorMessage = CheckForResponseInconsistency(cacheMetadata);

            if (inconsistencyErrorMessage != null)
            {
                return(new GetContentHashListResult(inconsistencyErrorMessage));
            }

            if (cacheMetadata?.ContentHashListWithDeterminism.ContentHashList == null)
            {
                // Miss
                return(new GetContentHashListResult(new ContentHashListWithDeterminism(null, CacheDeterminism.None)));
            }

            ContentHashList  contentHashList = cacheMetadata.ContentHashListWithDeterminism.ContentHashList;
            CacheDeterminism determinism     = UnpackDeterminism(cacheMetadata, cacheId);

            return(new GetContentHashListResult(new ContentHashListWithDeterminism(contentHashList, determinism)));
        }
Example #14
0
 /// <summary>
 /// Initializes a new instance of the <see cref="SelectorAndContentHashListWithCacheMetadata"/> struct.
 /// </summary>
 public SelectorAndContentHashListWithCacheMetadata(Selector selector, ContentHashListWithCacheMetadata contentHashList)
 {
     Selector        = selector;
     ContentHashList = contentHashList;
 }
        /// <inheritdoc />
        public async Task <ObjectResult <ContentHashListWithCacheMetadata> > AddContentHashListAsync(
            Context context,
            string cacheNamespace,
            StrongFingerprint strongFingerprint,
            ContentHashListWithCacheMetadata valueToAdd)
        {
            try
            {
                Func <System.IO.Stream, System.Threading.CancellationToken, Task <StructResult <ContentHash> > > putStreamFunc =
                    async(stream, cts) =>
                {
                    PutResult putResult = await _blobContentSession.PutStreamAsync(context, HashType.Vso0, stream, cts);

                    if (putResult.Succeeded)
                    {
                        return(new StructResult <ContentHash>(putResult.ContentHash));
                    }

                    return(new StructResult <ContentHash>(putResult));
                };

                StructResult <ContentHash> blobIdOfContentHashListResult =
                    await BlobContentHashListExtensions.PackInBlob(
                        putStreamFunc,
                        valueToAdd.ContentHashListWithDeterminism);

                if (!blobIdOfContentHashListResult.Succeeded)
                {
                    return(new ObjectResult <ContentHashListWithCacheMetadata>(blobIdOfContentHashListResult));
                }

                var blobContentHashListWithDeterminism =
                    new BlobContentHashListWithDeterminism(
                        valueToAdd.ContentHashListWithDeterminism.Determinism.EffectiveGuid,
                        BlobIdentifierToContentHashExtensions.ToBlobIdentifier(blobIdOfContentHashListResult.Data));

                var blobContentHashListWithCacheMetadata = new BlobContentHashListWithCacheMetadata(
                    blobContentHashListWithDeterminism,
                    valueToAdd.GetRawExpirationTimeUtc(),
                    valueToAdd.ContentGuarantee,
                    valueToAdd.HashOfExistingContentHashList);

                BlobContentHashListResponse addResult = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync(
                    context,
                    "AddContentHashList",
                    innerCts => _buildCacheHttpClient.AddContentHashListAsync(
                        cacheNamespace,
                        strongFingerprint,
                        blobContentHashListWithCacheMetadata),
                    CancellationToken.None).ConfigureAwait(false);

                DownloadUriCache.Instance.BulkAddDownloadUris(addResult.BlobDownloadUris);

                // add succeeded but returned an empty contenthashlistwith cache metadata. correct this.
                if (addResult.ContentHashListWithCacheMetadata == null)
                {
                    return
                        (new ObjectResult <ContentHashListWithCacheMetadata>(
                             new ContentHashListWithCacheMetadata(
                                 new ContentHashListWithDeterminism(null, blobContentHashListWithCacheMetadata.Determinism),
                                 blobContentHashListWithCacheMetadata.GetRawExpirationTimeUtc(),
                                 blobContentHashListWithCacheMetadata.ContentGuarantee)));
                }
                else
                {
                    return(await UnpackBlobContentHashListAsync(context, addResult.ContentHashListWithCacheMetadata));
                }
            }
            catch (Exception ex)
            {
                return(new ObjectResult <ContentHashListWithCacheMetadata>(ex));
            }
        }
        private void SealIfNecessaryAfterGet(Context context, StrongFingerprint strongFingerprint, ContentHashListWithCacheMetadata cacheMetadata)
        {
            if (WriteThroughContentSession == null)
            {
                return;
            }

            if (cacheMetadata != null && cacheMetadata.GetEffectiveExpirationTimeUtc() == null)
            {
                // Value is unbacked in VSTS
                SealInTheBackground(context, strongFingerprint, cacheMetadata.ContentHashListWithDeterminism);
            }
        }
Example #17
0
        /// <inheritdoc />
        public Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync(
            Context context,
            StrongFingerprint strongFingerprint,
            ContentHashListWithDeterminism contentHashListWithDeterminism,
            CancellationToken cts,
            UrgencyHint urgencyHint)
        {
            return(new OperationContext(context, cts).PerformOperationAsync(
                       Tracer.MemoizationStoreTracer,
                       async() =>
            {
                // TODO: Split this out into separate implementations for WriteThrough vs. WriteBehind (bug 1365340)
                if (WriteThroughContentSession == null)
                {
                    ContentAvailabilityGuarantee guarantee =
                        ManuallyExtendContentLifetime
                                ? ContentAvailabilityGuarantee.NoContentBackedByCache
                                : ContentAvailabilityGuarantee.AllContentBackedByCache;

                    return await AddOrGetContentHashListAsync(
                        context,
                        strongFingerprint,
                        contentHashListWithDeterminism,
                        guarantee).ConfigureAwait(false);
                }

                // Ensure that the content exists somewhere before trying to add
                if (!await EnsureContentIsAvailableAsync(
                        context, contentHashListWithDeterminism.ContentHashList.Hashes, cts, urgencyHint).ConfigureAwait(false))
                {
                    return new AddOrGetContentHashListResult(
                        "Referenced content must exist in the cache before a new content hash list is added.");
                }

                DateTime expirationUtc = FingerprintTracker.GenerateNewExpiration();
                var valueToAdd = new ContentHashListWithCacheMetadata(
                    contentHashListWithDeterminism,
                    expirationUtc,
                    ContentAvailabilityGuarantee.NoContentBackedByCache);

                DateTime?rawExpiration = null;
                const int addLimit = 3;
                for (int addAttempts = 0; addAttempts < addLimit; addAttempts++)
                {
                    var debugString = $"Adding contentHashList=[{valueToAdd.ContentHashListWithDeterminism.ContentHashList}] " +
                                      $"determinism=[{valueToAdd.ContentHashListWithDeterminism.Determinism}] to VSTS with " +
                                      $"contentAvailabilityGuarantee=[{valueToAdd.ContentGuarantee}], expirationUtc=[{expirationUtc}], forceUpdate=[{ForceUpdateOnAddContentHashList}]";
                    Tracer.Debug(context, debugString);
                    Result <ContentHashListWithCacheMetadata> responseObject =
                        await ContentHashListAdapter.AddContentHashListAsync(
                            context,
                            CacheNamespace,
                            strongFingerprint,
                            valueToAdd,
                            forceUpdate: ForceUpdateOnAddContentHashList).ConfigureAwait(false);

                    if (!responseObject.Succeeded)
                    {
                        return new AddOrGetContentHashListResult(responseObject);
                    }

                    ContentHashListWithCacheMetadata response = responseObject.Value;
                    var inconsistencyErrorMessage = CheckForResponseInconsistency(response);
                    if (inconsistencyErrorMessage != null)
                    {
                        return new AddOrGetContentHashListResult(inconsistencyErrorMessage);
                    }

                    rawExpiration = response.GetRawExpirationTimeUtc();

                    ContentHashList contentHashListToReturn =
                        UnpackContentHashListAfterAdd(contentHashListWithDeterminism.ContentHashList, response);
                    CacheDeterminism determinismToReturn = UnpackDeterminism(response, CacheId);

                    bool needToUpdateExistingValue = await CheckNeedToUpdateExistingValueAsync(
                        context,
                        response,
                        contentHashListToReturn,
                        cts,
                        urgencyHint).ConfigureAwait(false);
                    if (!needToUpdateExistingValue)
                    {
                        SealIfNecessaryAfterUnbackedAddOrGet(context, strongFingerprint, contentHashListWithDeterminism, response);

                        await TrackFingerprintAsync(context, strongFingerprint, rawExpiration).ConfigureAwait(false);
                        return new AddOrGetContentHashListResult(
                            new ContentHashListWithDeterminism(contentHashListToReturn, determinismToReturn));
                    }

                    var hashOfExistingContentHashList = response.HashOfExistingContentHashList;
                    Tracer.Debug(context, $"Attempting to replace unbacked value with hash {hashOfExistingContentHashList.ToHex()}");
                    valueToAdd = new ContentHashListWithCacheMetadata(
                        contentHashListWithDeterminism,
                        expirationUtc,
                        ContentAvailabilityGuarantee.NoContentBackedByCache,
                        hashOfExistingContentHashList
                        );
                }

                Tracer.Warning(
                    context,
                    $"Lost the AddOrUpdate race {addLimit} times against unbacked values. Returning as though the add succeeded for now.");
                await TrackFingerprintAsync(context, strongFingerprint, rawExpiration).ConfigureAwait(false);
                return new AddOrGetContentHashListResult(new ContentHashListWithDeterminism(null, CacheDeterminism.None));
            },
                       traceOperationStarted: true,
                       extraStartMessage: $"StrongFingerprint=({strongFingerprint}), ForceUpdate=({ForceUpdateOnAddContentHashList}) {contentHashListWithDeterminism.ToTraceString()}",
                       extraEndMessage: _ => $"StrongFingerprint=({strongFingerprint}), ForceUpdate=({ForceUpdateOnAddContentHashList}) {contentHashListWithDeterminism.ToTraceString()}"));
        }