/// <inheritdoc />
public Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync(Context context, StrongFingerprint strongFingerprint, ContentHashListWithDeterminism contentHashListWithDeterminism, CancellationToken cts, UrgencyHint urgencyHint)
{
return(WithOperationContext(
context,
cts,
operationContext =>
{
return operationContext.PerformOperationAsync(
DistributedTracer,
async() =>
{
// Metadata cache assumes no guarantees about the fingerprints added, hence invalidate the cache and serve the request using backing store
var cacheResult = await MetadataCache.DeleteFingerprintAsync(context, strongFingerprint);
if (!cacheResult.Succeeded)
{
context.Error($"Error while removing fingerprint {strongFingerprint} from metadata cache. Result: {cacheResult}.");
}
return await _innerCacheSession.AddOrGetContentHashListAsync(context, strongFingerprint, contentHashListWithDeterminism, cts, urgencyHint);
},
traceOperationStarted: true,
extraStartMessage: $"StrongFingerprint=({strongFingerprint}) {contentHashListWithDeterminism.ToTraceString()}",
extraEndMessage: _ => $"StrongFingerprint=({strongFingerprint}) {contentHashListWithDeterminism.ToTraceString()}");
}));
}
public Task <BoolResult> PublishContentHashListAsync(
Context context,
StrongFingerprint fingerprint,
ContentHashListWithDeterminism contentHashList,
CancellationToken token)
{
Contract.Check(_publisher != null)?.Assert("Startup should be run before attempting to publish.");
var operationContext = new OperationContext(context, token);
Tracer.Debug(operationContext, $"Enqueueing publish request for StrongFingerprint=[{fingerprint}], CHL=[{contentHashList.ToTraceString()}]");
return(_publishingGate.GatedOperationAsync(
(timeSpentWaiting, gateCount) =>
{
ContentHashList?hashListInRemote = null;
return operationContext.PerformOperationAsync(
Tracer,
async() =>
{
// Make sure to push the blob in the selector if it exists.
var hashesToPush = new List <ContentHash>(contentHashList.ContentHashList.Hashes);
if (!fingerprint.Selector.ContentHash.IsZero())
{
hashesToPush.Add(fingerprint.Selector.ContentHash);
}
var remotePinResults = await Task.WhenAll(await _publisher.PinAsync(operationContext, hashesToPush, token));
var missingFromRemote = remotePinResults
.Where(r => !r.Item.Succeeded)
.Select(r => hashesToPush[r.Index])
.ToArray();
if (missingFromRemote.Length > 0)
{
await PushToRemoteAsync(operationContext, missingFromRemote).ThrowIfFailure();
}
var addOrGetResult = await _publisher.AddOrGetContentHashListAsync(operationContext, fingerprint, contentHashList, token).ThrowIfFailure();
hashListInRemote = addOrGetResult.ContentHashListWithDeterminism.ContentHashList;
return BoolResult.Success;
},
traceOperationStarted: false,
extraEndMessage: result =>
$"Added=[{result.Succeeded && hashListInRemote is null}], " +
$"StrongFingerprint=[{fingerprint}], " +
$"ContentHashList=[{contentHashList.ToTraceString()}], " +
$"TimeSpentWaiting=[{timeSpentWaiting}], " +
$"GateCount=[{gateCount}]");
},
token));
}
/// <summary>
/// Performs a compare exchange operation on metadata, while ensuring all invariants are kept. If the
/// fingerprint is not present, then it is inserted.
/// </summary>
public Task <Result <bool> > CompareExchange(
OperationContext context,
StrongFingerprint strongFingerprint,
string expectedReplacementToken,
ContentHashListWithDeterminism expected,
ContentHashListWithDeterminism replacement)
{
return(context.PerformOperationAsync(
Tracer,
() => CompareExchangeCore(context, strongFingerprint, expectedReplacementToken, expected, replacement),
extraStartMessage: $"StrongFingerprint=({strongFingerprint}) expected=[{expected.ToTraceString()}] replacement=[{replacement.ToTraceString()}]",
extraEndMessage: _ => $"StrongFingerprint=({strongFingerprint}) expected=[{expected.ToTraceString()}] replacement=[{replacement.ToTraceString()}]"));
}
/// <inheritdoc />
public Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync(
Context context,
StrongFingerprint strongFingerprint,
ContentHashListWithDeterminism contentHashListWithDeterminism,
CancellationToken cts,
UrgencyHint urgencyHint)
{
return(new OperationContext(context, cts).PerformOperationAsync(
Tracer.MemoizationStoreTracer,
async() =>
{
// TODO: Split this out into separate implementations for WriteThrough vs. WriteBehind (bug 1365340)
if (WriteThroughContentSession == null)
{
ContentAvailabilityGuarantee guarantee =
ManuallyExtendContentLifetime
? ContentAvailabilityGuarantee.NoContentBackedByCache
: ContentAvailabilityGuarantee.AllContentBackedByCache;
return await AddOrGetContentHashListAsync(
context,
strongFingerprint,
contentHashListWithDeterminism,
guarantee).ConfigureAwait(false);
}
// Ensure that the content exists somewhere before trying to add
if (!await EnsureContentIsAvailableAsync(
context, contentHashListWithDeterminism.ContentHashList.Hashes, cts, urgencyHint).ConfigureAwait(false))
{
return new AddOrGetContentHashListResult(
"Referenced content must exist in the cache before a new content hash list is added.");
}
DateTime expirationUtc = FingerprintTracker.GenerateNewExpiration();
var valueToAdd = new ContentHashListWithCacheMetadata(
contentHashListWithDeterminism,
expirationUtc,
ContentAvailabilityGuarantee.NoContentBackedByCache);
DateTime?rawExpiration = null;
const int addLimit = 3;
for (int addAttempts = 0; addAttempts < addLimit; addAttempts++)
{
var debugString = $"Adding contentHashList=[{valueToAdd.ContentHashListWithDeterminism.ContentHashList}] " +
$"determinism=[{valueToAdd.ContentHashListWithDeterminism.Determinism}] to VSTS with " +
$"contentAvailabilityGuarantee=[{valueToAdd.ContentGuarantee}], expirationUtc=[{expirationUtc}], forceUpdate=[{ForceUpdateOnAddContentHashList}]";
Tracer.Debug(context, debugString);
Result <ContentHashListWithCacheMetadata> responseObject =
await ContentHashListAdapter.AddContentHashListAsync(
context,
CacheNamespace,
strongFingerprint,
valueToAdd,
forceUpdate: ForceUpdateOnAddContentHashList).ConfigureAwait(false);
if (!responseObject.Succeeded)
{
return new AddOrGetContentHashListResult(responseObject);
}
ContentHashListWithCacheMetadata response = responseObject.Value;
var inconsistencyErrorMessage = CheckForResponseInconsistency(response);
if (inconsistencyErrorMessage != null)
{
return new AddOrGetContentHashListResult(inconsistencyErrorMessage);
}
rawExpiration = response.GetRawExpirationTimeUtc();
ContentHashList contentHashListToReturn =
UnpackContentHashListAfterAdd(contentHashListWithDeterminism.ContentHashList, response);
CacheDeterminism determinismToReturn = UnpackDeterminism(response, CacheId);
bool needToUpdateExistingValue = await CheckNeedToUpdateExistingValueAsync(
context,
response,
contentHashListToReturn,
cts,
urgencyHint).ConfigureAwait(false);
if (!needToUpdateExistingValue)
{
SealIfNecessaryAfterUnbackedAddOrGet(context, strongFingerprint, contentHashListWithDeterminism, response);
await TrackFingerprintAsync(context, strongFingerprint, rawExpiration).ConfigureAwait(false);
return new AddOrGetContentHashListResult(
new ContentHashListWithDeterminism(contentHashListToReturn, determinismToReturn));
}
var hashOfExistingContentHashList = response.HashOfExistingContentHashList;
Tracer.Debug(context, $"Attempting to replace unbacked value with hash {hashOfExistingContentHashList.ToHex()}");
valueToAdd = new ContentHashListWithCacheMetadata(
contentHashListWithDeterminism,
expirationUtc,
ContentAvailabilityGuarantee.NoContentBackedByCache,
hashOfExistingContentHashList
);
}
Tracer.Warning(
context,
$"Lost the AddOrUpdate race {addLimit} times against unbacked values. Returning as though the add succeeded for now.");
await TrackFingerprintAsync(context, strongFingerprint, rawExpiration).ConfigureAwait(false);
return new AddOrGetContentHashListResult(new ContentHashListWithDeterminism(null, CacheDeterminism.None));
},
traceOperationStarted: true,
extraStartMessage: $"StrongFingerprint=({strongFingerprint}), ForceUpdate=({ForceUpdateOnAddContentHashList}) {contentHashListWithDeterminism.ToTraceString()}",
extraEndMessage: _ => $"StrongFingerprint=({strongFingerprint}), ForceUpdate=({ForceUpdateOnAddContentHashList}) {contentHashListWithDeterminism.ToTraceString()}"));
}
/// <summary>
/// Performs a compare exchange operation on metadata, while ensuring all invariants are kept. If the
/// fingerprint is not present, then it is inserted.
/// </summary>
public Task <Result <bool> > CompareExchange(
OperationContext context,
StrongFingerprint strongFingerprint,
string expectedReplacementToken,
ContentHashListWithDeterminism expected,
ContentHashListWithDeterminism replacement)
{
return(context.PerformOperationWithTimeoutAsync(
Tracer,
nestedContext => CompareExchangeCore(nestedContext, strongFingerprint, expectedReplacementToken, expected, replacement),
extraStartMessage: $"StrongFingerprint=[{strongFingerprint}] ExpectedReplacementToken=[{expectedReplacementToken}] Expected=[{expected.ToTraceString()}] Replacement=[{replacement.ToTraceString()}]",
extraEndMessage: result => $"StrongFingerprint=[{strongFingerprint}] ExpectedReplacementToken=[{expectedReplacementToken}] Expected=[{expected.ToTraceString()}] Replacement=[{replacement.ToTraceString()}] Exchanged=[{result.GetValueOrDefault(false)}]",
timeout: _timeout));
}
/// <inheritdoc />
public Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync(Context context, StrongFingerprint strongFingerprint, ContentHashListWithDeterminism contentHashListWithDeterminism, CancellationToken cts, UrgencyHint urgencyHint = UrgencyHint.Nominal)
{
var contentHashes = contentHashListWithDeterminism.ContentHashList.Hashes;
return(PerformOperationAsync(
context,
cts,
(ctx, client) => client.AddOrGetContentHashListAsync(ctx, strongFingerprint, contentHashListWithDeterminism),
counter: _memoizationCounters[MemoizationStoreCounters.AddOrGetContentHashList],
retryCounter: _memoizationCounters[MemoizationStoreCounters.AddOrGetContentHashListRetries],
additionalStartMessage: $"StrongFingerprint=({strongFingerprint}) {contentHashListWithDeterminism.ToTraceString()}",
additionalStopMessage: $"StrongFingerprint=({strongFingerprint}) {contentHashListWithDeterminism.ToTraceString()}"));
}