/// <inheritdoc />
public async Task <ObjectResult <ContentHashListWithCacheMetadata> > AddContentHashListAsync(
Context context,
string cacheNamespace,
StrongFingerprint strongFingerprint,
ContentHashListWithCacheMetadata valueToAdd,
bool forceUpdate)
{
try
{
ContentHashListResponse addResult = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync(
context,
"AddContentHashList",
innerCts => _buildCacheHttpClient.AddContentHashListAsync(
cacheNamespace,
strongFingerprint,
valueToAdd,
forceUpdate), CancellationToken.None).ConfigureAwait(false);
// The return value is null if the server fails adding content hash list to the backing store.
// See BuildCacheService.AddContentHashListAsync for more details about the implementation invariants/guarantees.
if (addResult != null)
{
DownloadUriCache.Instance.BulkAddDownloadUris(addResult.BlobDownloadUris);
}
// add succeeded but returned an empty contenthashlistwith cache metadata. correct this.
if (addResult?.ContentHashListWithCacheMetadata == null)
{
return
(new ObjectResult <ContentHashListWithCacheMetadata>(
new ContentHashListWithCacheMetadata(
new ContentHashListWithDeterminism(null, valueToAdd.ContentHashListWithDeterminism.Determinism),
valueToAdd.GetRawExpirationTimeUtc(),
valueToAdd.ContentGuarantee,
valueToAdd.HashOfExistingContentHashList)));
}
else if (addResult.ContentHashListWithCacheMetadata.ContentHashListWithDeterminism.ContentHashList != null &&
addResult.ContentHashListWithCacheMetadata.HashOfExistingContentHashList == null)
{
return(new ObjectResult <ContentHashListWithCacheMetadata>(
new ContentHashListWithCacheMetadata(
addResult.ContentHashListWithCacheMetadata.ContentHashListWithDeterminism,
addResult.ContentHashListWithCacheMetadata.GetRawExpirationTimeUtc(),
addResult.ContentHashListWithCacheMetadata.ContentGuarantee,
addResult.ContentHashListWithCacheMetadata.ContentHashListWithDeterminism.ContentHashList.GetHashOfHashes())));
}
else
{
return(new ObjectResult <ContentHashListWithCacheMetadata>(addResult.ContentHashListWithCacheMetadata));
}
}
catch (Exception ex)
{
return(new ObjectResult <ContentHashListWithCacheMetadata>(ex));
}
}
/// <inheritdoc />
public Task <GetContentHashListResult> GetContentHashListAsync(
Context context,
StrongFingerprint strongFingerprint,
CancellationToken cts,
UrgencyHint urgencyHint)
{
return(GetContentHashListCall.RunAsync(Tracer.MemoizationStoreTracer, context, strongFingerprint, async() =>
{
// Check for pre-fetched data
ContentHashListWithDeterminism contentHashListWithDeterminism;
if (ContentHashListWithDeterminismCache.Instance.TryGetValue(
CacheNamespace, strongFingerprint, out contentHashListWithDeterminism))
{
Tracer.RecordUseOfPrefetchedContentHashList();
FingerprintTracker.Track(
strongFingerprint,
contentHashListWithDeterminism.Determinism.ExpirationUtc);
return new GetContentHashListResult(contentHashListWithDeterminism);
}
// No pre-fetched data. Need to query the server.
ObjectResult <ContentHashListWithCacheMetadata> responseObject =
await ContentHashListAdapter.GetContentHashListAsync(context, CacheNamespace, strongFingerprint).ConfigureAwait(false);
if (!responseObject.Succeeded)
{
return new GetContentHashListResult(responseObject);
}
ContentHashListWithCacheMetadata response = responseObject.Data;
if (response.ContentHashListWithDeterminism.ContentHashList == null)
{
// Miss
return new GetContentHashListResult(new ContentHashListWithDeterminism(null, CacheDeterminism.None));
}
GetContentHashListResult unpackResult = UnpackContentHashListWithDeterminismAfterGet(response, CacheId);
if (!unpackResult.Succeeded)
{
return unpackResult;
}
SealIfNecessaryAfterGet(context, strongFingerprint, response);
FingerprintTracker.Track(strongFingerprint, response.GetRawExpirationTimeUtc());
return new GetContentHashListResult(unpackResult.ContentHashListWithDeterminism);
}));
}
/// <inheritdoc />
public Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync(
Context context,
StrongFingerprint strongFingerprint,
ContentHashListWithDeterminism contentHashListWithDeterminism,
CancellationToken cts,
UrgencyHint urgencyHint)
{
return(new OperationContext(context, cts).PerformOperationAsync(
Tracer.MemoizationStoreTracer,
async() =>
{
// TODO: Split this out into separate implementations for WriteThrough vs. WriteBehind (bug 1365340)
if (WriteThroughContentSession == null)
{
ContentAvailabilityGuarantee guarantee =
ManuallyExtendContentLifetime
? ContentAvailabilityGuarantee.NoContentBackedByCache
: ContentAvailabilityGuarantee.AllContentBackedByCache;
return await AddOrGetContentHashListAsync(
context,
strongFingerprint,
contentHashListWithDeterminism,
guarantee).ConfigureAwait(false);
}
// Ensure that the content exists somewhere before trying to add
if (!await EnsureContentIsAvailableAsync(
context, contentHashListWithDeterminism.ContentHashList.Hashes, cts, urgencyHint).ConfigureAwait(false))
{
return new AddOrGetContentHashListResult(
"Referenced content must exist in the cache before a new content hash list is added.");
}
DateTime expirationUtc = FingerprintTracker.GenerateNewExpiration();
var valueToAdd = new ContentHashListWithCacheMetadata(
contentHashListWithDeterminism,
expirationUtc,
ContentAvailabilityGuarantee.NoContentBackedByCache);
DateTime?rawExpiration = null;
const int addLimit = 3;
for (int addAttempts = 0; addAttempts < addLimit; addAttempts++)
{
var debugString = $"Adding contentHashList=[{valueToAdd.ContentHashListWithDeterminism.ContentHashList}] " +
$"determinism=[{valueToAdd.ContentHashListWithDeterminism.Determinism}] to VSTS with " +
$"contentAvailabilityGuarantee=[{valueToAdd.ContentGuarantee}], expirationUtc=[{expirationUtc}], forceUpdate=[{ForceUpdateOnAddContentHashList}]";
Tracer.Debug(context, debugString);
Result <ContentHashListWithCacheMetadata> responseObject =
await ContentHashListAdapter.AddContentHashListAsync(
context,
CacheNamespace,
strongFingerprint,
valueToAdd,
forceUpdate: ForceUpdateOnAddContentHashList).ConfigureAwait(false);
if (!responseObject.Succeeded)
{
return new AddOrGetContentHashListResult(responseObject);
}
ContentHashListWithCacheMetadata response = responseObject.Value;
var inconsistencyErrorMessage = CheckForResponseInconsistency(response);
if (inconsistencyErrorMessage != null)
{
return new AddOrGetContentHashListResult(inconsistencyErrorMessage);
}
rawExpiration = response.GetRawExpirationTimeUtc();
ContentHashList contentHashListToReturn =
UnpackContentHashListAfterAdd(contentHashListWithDeterminism.ContentHashList, response);
CacheDeterminism determinismToReturn = UnpackDeterminism(response, CacheId);
bool needToUpdateExistingValue = await CheckNeedToUpdateExistingValueAsync(
context,
response,
contentHashListToReturn,
cts,
urgencyHint).ConfigureAwait(false);
if (!needToUpdateExistingValue)
{
SealIfNecessaryAfterUnbackedAddOrGet(context, strongFingerprint, contentHashListWithDeterminism, response);
await TrackFingerprintAsync(context, strongFingerprint, rawExpiration).ConfigureAwait(false);
return new AddOrGetContentHashListResult(
new ContentHashListWithDeterminism(contentHashListToReturn, determinismToReturn));
}
var hashOfExistingContentHashList = response.HashOfExistingContentHashList;
Tracer.Debug(context, $"Attempting to replace unbacked value with hash {hashOfExistingContentHashList.ToHex()}");
valueToAdd = new ContentHashListWithCacheMetadata(
contentHashListWithDeterminism,
expirationUtc,
ContentAvailabilityGuarantee.NoContentBackedByCache,
hashOfExistingContentHashList
);
}
Tracer.Warning(
context,
$"Lost the AddOrUpdate race {addLimit} times against unbacked values. Returning as though the add succeeded for now.");
await TrackFingerprintAsync(context, strongFingerprint, rawExpiration).ConfigureAwait(false);
return new AddOrGetContentHashListResult(new ContentHashListWithDeterminism(null, CacheDeterminism.None));
},
traceOperationStarted: true,
extraStartMessage: $"StrongFingerprint=({strongFingerprint}), ForceUpdate=({ForceUpdateOnAddContentHashList}) {contentHashListWithDeterminism.ToTraceString()}",
extraEndMessage: _ => $"StrongFingerprint=({strongFingerprint}), ForceUpdate=({ForceUpdateOnAddContentHashList}) {contentHashListWithDeterminism.ToTraceString()}"));
}
/// <inheritdoc />
public async Task <ObjectResult <ContentHashListWithCacheMetadata> > AddContentHashListAsync(
Context context,
string cacheNamespace,
StrongFingerprint strongFingerprint,
ContentHashListWithCacheMetadata valueToAdd)
{
try
{
Func <System.IO.Stream, System.Threading.CancellationToken, Task <StructResult <ContentHash> > > putStreamFunc =
async(stream, cts) =>
{
PutResult putResult = await _blobContentSession.PutStreamAsync(context, HashType.Vso0, stream, cts);
if (putResult.Succeeded)
{
return(new StructResult <ContentHash>(putResult.ContentHash));
}
return(new StructResult <ContentHash>(putResult));
};
StructResult <ContentHash> blobIdOfContentHashListResult =
await BlobContentHashListExtensions.PackInBlob(
putStreamFunc,
valueToAdd.ContentHashListWithDeterminism);
if (!blobIdOfContentHashListResult.Succeeded)
{
return(new ObjectResult <ContentHashListWithCacheMetadata>(blobIdOfContentHashListResult));
}
var blobContentHashListWithDeterminism =
new BlobContentHashListWithDeterminism(
valueToAdd.ContentHashListWithDeterminism.Determinism.EffectiveGuid,
BlobIdentifierToContentHashExtensions.ToBlobIdentifier(blobIdOfContentHashListResult.Data));
var blobContentHashListWithCacheMetadata = new BlobContentHashListWithCacheMetadata(
blobContentHashListWithDeterminism,
valueToAdd.GetRawExpirationTimeUtc(),
valueToAdd.ContentGuarantee,
valueToAdd.HashOfExistingContentHashList);
BlobContentHashListResponse addResult = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync(
context,
"AddContentHashList",
innerCts => _buildCacheHttpClient.AddContentHashListAsync(
cacheNamespace,
strongFingerprint,
blobContentHashListWithCacheMetadata),
CancellationToken.None).ConfigureAwait(false);
DownloadUriCache.Instance.BulkAddDownloadUris(addResult.BlobDownloadUris);
// add succeeded but returned an empty contenthashlistwith cache metadata. correct this.
if (addResult.ContentHashListWithCacheMetadata == null)
{
return
(new ObjectResult <ContentHashListWithCacheMetadata>(
new ContentHashListWithCacheMetadata(
new ContentHashListWithDeterminism(null, blobContentHashListWithCacheMetadata.Determinism),
blobContentHashListWithCacheMetadata.GetRawExpirationTimeUtc(),
blobContentHashListWithCacheMetadata.ContentGuarantee)));
}
else
{
return(await UnpackBlobContentHashListAsync(context, addResult.ContentHashListWithCacheMetadata));
}
}
catch (Exception ex)
{
return(new ObjectResult <ContentHashListWithCacheMetadata>(ex));
}
}