/// <inheritdoc />
public Task <BoolResult> ShutdownAsync(Context context)
{
ShutdownStarted = true;
return(ShutdownCall <MemoizationStoreTracer> .RunAsync(Tracer.MemoizationStoreTracer, context, async() =>
{
Tracer.Debug(context, "IncorporateOnShutdown start");
Tracer.Debug(context, $"Incorporate fingerprints feature enabled:[{_fingerprintIncorporationEnabled}]");
Tracer.Debug(context, $"Total fingerprints to be incorporated:[{FingerprintTracker.Count}]");
Tracer.Debug(context, $"Max fingerprints per incorporate request(=chunk size):[{_maxFingerprintsPerIncorporateRequest}]");
Tracer.Debug(context, $"Max incorporate requests allowed in parallel:[{_maxDegreeOfParallelismForIncorporateRequests}]");
if (_fingerprintIncorporationEnabled)
{
// Incorporating all of the fingerprints for a build, in one request, to a single endpoint causes pain. Incorporation involves
// extending the lifetime of all fingerprints *and* content/s mapped to each fingerprint. Processing a large request payload
// results in, potentially, fanning out a massive number of "lifetime extend" requests to itemstore and blobstore, which can
// bring down the endpoint. Break this down into chunks so that multiple, load-balanced endpoints can share the burden.
List <StrongFingerprint> fingerprintsToBump = FingerprintTracker.StaleFingerprints.ToList();
Tracer.Debug(context, $"Total fingerprints to be sent in incorporation requeststo the service: {fingerprintsToBump.Count}");
List <List <StrongFingerprintAndExpiration> > chunks = fingerprintsToBump.Select(
strongFingerprint => new StrongFingerprintAndExpiration(strongFingerprint, FingerprintTracker.GenerateNewExpiration())
).GetPages(_maxFingerprintsPerIncorporateRequest).ToList();
Tracer.Debug(context, $"Total fingerprint incorporation requests to be issued(=number of fingerprint chunks):[{chunks.Count}]");
var incorporateBlock = new ActionBlock <IEnumerable <StrongFingerprintAndExpiration> >(
async chunk =>
{
await ContentHashListAdapter.IncorporateStrongFingerprints(
context,
CacheNamespace,
new IncorporateStrongFingerprintsRequest(chunk.ToList().AsReadOnly())
).ConfigureAwait(false);
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = _maxDegreeOfParallelismForIncorporateRequests
});
foreach (var chunk in chunks)
{
await incorporateBlock.SendAsync(chunk);
}
incorporateBlock.Complete();
await incorporateBlock.Completion.ConfigureAwait(false); // TODO: Gracefully handle exceptions so that the rest of shutdown can happen (bug 1365340)
Tracer.Debug(context, "IncorporateOnShutdown stop");
}
if (_taskTracker != null)
{
await _taskTracker.Synchronize().ConfigureAwait(false);
await _taskTracker.ShutdownAsync(context).ConfigureAwait(false);
}
var backingContentSessionTask = Task.Run(async() => await BackingContentSession.ShutdownAsync(context).ConfigureAwait(false));
var writeThroughContentSessionResult = WriteThroughContentSession != null
? await WriteThroughContentSession.ShutdownAsync(context).ConfigureAwait(false)
: BoolResult.Success;
var backingContentSessionResult = await backingContentSessionTask.ConfigureAwait(false);
BoolResult result;
if (backingContentSessionResult.Succeeded && writeThroughContentSessionResult.Succeeded)
{
if (_sealingErrorsToPrintOnShutdown.Any())
{
var sb = new StringBuilder();
sb.AppendLine("Error(s) during background sealing:");
foreach (var sealingError in _sealingErrorsToPrintOnShutdown)
{
sb.AppendLine($"[{sealingError}]");
}
if (_sealingErrorCount > MaxSealingErrorsToPrintOnShutdown)
{
sb.AppendLine($"See log for the other {MaxSealingErrorsToPrintOnShutdown - _sealingErrorCount} error(s).");
}
result = new BoolResult(sb.ToString());
}
else
{
result = BoolResult.Success;
}
}
else
{
var sb = new StringBuilder();
if (!backingContentSessionResult.Succeeded)
{
sb.Append($"Backing content session shutdown failed, error=[{backingContentSessionResult}]");
}
if (!writeThroughContentSessionResult.Succeeded)
{
sb.Append(sb.Length > 0 ? ", " : string.Empty);
sb.Append($"Write-through content session shutdown failed, error=[{writeThroughContentSessionResult}]");
}
result = new BoolResult(sb.ToString());
}
ShutdownCompleted = true;
return result;
}));
}
/// <inheritdoc />
public Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync(
Context context,
StrongFingerprint strongFingerprint,
ContentHashListWithDeterminism contentHashListWithDeterminism,
CancellationToken cts,
UrgencyHint urgencyHint)
{
return(new OperationContext(context, cts).PerformOperationAsync(
Tracer.MemoizationStoreTracer,
async() =>
{
// TODO: Split this out into separate implementations for WriteThrough vs. WriteBehind (bug 1365340)
if (WriteThroughContentSession == null)
{
ContentAvailabilityGuarantee guarantee =
ManuallyExtendContentLifetime
? ContentAvailabilityGuarantee.NoContentBackedByCache
: ContentAvailabilityGuarantee.AllContentBackedByCache;
return await AddOrGetContentHashListAsync(
context,
strongFingerprint,
contentHashListWithDeterminism,
guarantee).ConfigureAwait(false);
}
// Ensure that the content exists somewhere before trying to add
if (!await EnsureContentIsAvailableAsync(
context, contentHashListWithDeterminism.ContentHashList.Hashes, cts, urgencyHint).ConfigureAwait(false))
{
return new AddOrGetContentHashListResult(
"Referenced content must exist in the cache before a new content hash list is added.");
}
DateTime expirationUtc = FingerprintTracker.GenerateNewExpiration();
var valueToAdd = new ContentHashListWithCacheMetadata(
contentHashListWithDeterminism,
expirationUtc,
ContentAvailabilityGuarantee.NoContentBackedByCache);
DateTime?rawExpiration = null;
const int addLimit = 3;
for (int addAttempts = 0; addAttempts < addLimit; addAttempts++)
{
var debugString = $"Adding contentHashList=[{valueToAdd.ContentHashListWithDeterminism.ContentHashList}] " +
$"determinism=[{valueToAdd.ContentHashListWithDeterminism.Determinism}] to VSTS with " +
$"contentAvailabilityGuarantee=[{valueToAdd.ContentGuarantee}], expirationUtc=[{expirationUtc}], forceUpdate=[{ForceUpdateOnAddContentHashList}]";
Tracer.Debug(context, debugString);
Result <ContentHashListWithCacheMetadata> responseObject =
await ContentHashListAdapter.AddContentHashListAsync(
context,
CacheNamespace,
strongFingerprint,
valueToAdd,
forceUpdate: ForceUpdateOnAddContentHashList).ConfigureAwait(false);
if (!responseObject.Succeeded)
{
return new AddOrGetContentHashListResult(responseObject);
}
ContentHashListWithCacheMetadata response = responseObject.Value;
var inconsistencyErrorMessage = CheckForResponseInconsistency(response);
if (inconsistencyErrorMessage != null)
{
return new AddOrGetContentHashListResult(inconsistencyErrorMessage);
}
rawExpiration = response.GetRawExpirationTimeUtc();
ContentHashList contentHashListToReturn =
UnpackContentHashListAfterAdd(contentHashListWithDeterminism.ContentHashList, response);
CacheDeterminism determinismToReturn = UnpackDeterminism(response, CacheId);
bool needToUpdateExistingValue = await CheckNeedToUpdateExistingValueAsync(
context,
response,
contentHashListToReturn,
cts,
urgencyHint).ConfigureAwait(false);
if (!needToUpdateExistingValue)
{
SealIfNecessaryAfterUnbackedAddOrGet(context, strongFingerprint, contentHashListWithDeterminism, response);
await TrackFingerprintAsync(context, strongFingerprint, rawExpiration).ConfigureAwait(false);
return new AddOrGetContentHashListResult(
new ContentHashListWithDeterminism(contentHashListToReturn, determinismToReturn));
}
var hashOfExistingContentHashList = response.HashOfExistingContentHashList;
Tracer.Debug(context, $"Attempting to replace unbacked value with hash {hashOfExistingContentHashList.ToHex()}");
valueToAdd = new ContentHashListWithCacheMetadata(
contentHashListWithDeterminism,
expirationUtc,
ContentAvailabilityGuarantee.NoContentBackedByCache,
hashOfExistingContentHashList
);
}
Tracer.Warning(
context,
$"Lost the AddOrUpdate race {addLimit} times against unbacked values. Returning as though the add succeeded for now.");
await TrackFingerprintAsync(context, strongFingerprint, rawExpiration).ConfigureAwait(false);
return new AddOrGetContentHashListResult(new ContentHashListWithDeterminism(null, CacheDeterminism.None));
},
traceOperationStarted: true,
extraStartMessage: $"StrongFingerprint=({strongFingerprint}), ForceUpdate=({ForceUpdateOnAddContentHashList}) {contentHashListWithDeterminism.ToTraceString()}",
extraEndMessage: _ => $"StrongFingerprint=({strongFingerprint}), ForceUpdate=({ForceUpdateOnAddContentHashList}) {contentHashListWithDeterminism.ToTraceString()}"));
}
