/// <inheritdoc /> public Task <BoolResult> ShutdownAsync(Context context) { ShutdownStarted = true; return(ShutdownCall <MemoizationStoreTracer> .RunAsync(Tracer.MemoizationStoreTracer, context, async() => { Tracer.Debug(context, "IncorporateOnShutdown start"); Tracer.Debug(context, $"Incorporate fingerprints feature enabled:[{_fingerprintIncorporationEnabled}]"); Tracer.Debug(context, $"Total fingerprints to be incorporated:[{FingerprintTracker.Count}]"); Tracer.Debug(context, $"Max fingerprints per incorporate request(=chunk size):[{_maxFingerprintsPerIncorporateRequest}]"); Tracer.Debug(context, $"Max incorporate requests allowed in parallel:[{_maxDegreeOfParallelismForIncorporateRequests}]"); if (_fingerprintIncorporationEnabled) { // Incorporating all of the fingerprints for a build, in one request, to a single endpoint causes pain. Incorporation involves // extending the lifetime of all fingerprints *and* content/s mapped to each fingerprint. Processing a large request payload // results in, potentially, fanning out a massive number of "lifetime extend" requests to itemstore and blobstore, which can // bring down the endpoint. Break this down into chunks so that multiple, load-balanced endpoints can share the burden. List <StrongFingerprint> fingerprintsToBump = FingerprintTracker.StaleFingerprints.ToList(); Tracer.Debug(context, $"Total fingerprints to be sent in incorporation requeststo the service: {fingerprintsToBump.Count}"); List <List <StrongFingerprintAndExpiration> > chunks = fingerprintsToBump.Select( strongFingerprint => new StrongFingerprintAndExpiration(strongFingerprint, FingerprintTracker.GenerateNewExpiration()) ).GetPages(_maxFingerprintsPerIncorporateRequest).ToList(); Tracer.Debug(context, $"Total fingerprint incorporation requests to be issued(=number of fingerprint chunks):[{chunks.Count}]"); var incorporateBlock = new ActionBlock <IEnumerable <StrongFingerprintAndExpiration> >( async chunk => { await ContentHashListAdapter.IncorporateStrongFingerprints( context, CacheNamespace, new IncorporateStrongFingerprintsRequest(chunk.ToList().AsReadOnly()) ).ConfigureAwait(false); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = _maxDegreeOfParallelismForIncorporateRequests }); foreach (var chunk in chunks) { await incorporateBlock.SendAsync(chunk); } incorporateBlock.Complete(); await incorporateBlock.Completion.ConfigureAwait(false); // TODO: Gracefully handle exceptions so that the rest of shutdown can happen (bug 1365340) Tracer.Debug(context, "IncorporateOnShutdown stop"); } if (_taskTracker != null) { await _taskTracker.Synchronize().ConfigureAwait(false); await _taskTracker.ShutdownAsync(context).ConfigureAwait(false); } var backingContentSessionTask = Task.Run(async() => await BackingContentSession.ShutdownAsync(context).ConfigureAwait(false)); var writeThroughContentSessionResult = WriteThroughContentSession != null ? await WriteThroughContentSession.ShutdownAsync(context).ConfigureAwait(false) : BoolResult.Success; var backingContentSessionResult = await backingContentSessionTask.ConfigureAwait(false); BoolResult result; if (backingContentSessionResult.Succeeded && writeThroughContentSessionResult.Succeeded) { if (_sealingErrorsToPrintOnShutdown.Any()) { var sb = new StringBuilder(); sb.AppendLine("Error(s) during background sealing:"); foreach (var sealingError in _sealingErrorsToPrintOnShutdown) { sb.AppendLine($"[{sealingError}]"); } if (_sealingErrorCount > MaxSealingErrorsToPrintOnShutdown) { sb.AppendLine($"See log for the other {MaxSealingErrorsToPrintOnShutdown - _sealingErrorCount} error(s)."); } result = new BoolResult(sb.ToString()); } else { result = BoolResult.Success; } } else { var sb = new StringBuilder(); if (!backingContentSessionResult.Succeeded) { sb.Append($"Backing content session shutdown failed, error=[{backingContentSessionResult}]"); } if (!writeThroughContentSessionResult.Succeeded) { sb.Append(sb.Length > 0 ? ", " : string.Empty); sb.Append($"Write-through content session shutdown failed, error=[{writeThroughContentSessionResult}]"); } result = new BoolResult(sb.ToString()); } ShutdownCompleted = true; return result; })); }
/// <inheritdoc /> public Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync( Context context, StrongFingerprint strongFingerprint, ContentHashListWithDeterminism contentHashListWithDeterminism, CancellationToken cts, UrgencyHint urgencyHint) { return(new OperationContext(context, cts).PerformOperationAsync( Tracer.MemoizationStoreTracer, async() => { // TODO: Split this out into separate implementations for WriteThrough vs. WriteBehind (bug 1365340) if (WriteThroughContentSession == null) { ContentAvailabilityGuarantee guarantee = ManuallyExtendContentLifetime ? ContentAvailabilityGuarantee.NoContentBackedByCache : ContentAvailabilityGuarantee.AllContentBackedByCache; return await AddOrGetContentHashListAsync( context, strongFingerprint, contentHashListWithDeterminism, guarantee).ConfigureAwait(false); } // Ensure that the content exists somewhere before trying to add if (!await EnsureContentIsAvailableAsync( context, contentHashListWithDeterminism.ContentHashList.Hashes, cts, urgencyHint).ConfigureAwait(false)) { return new AddOrGetContentHashListResult( "Referenced content must exist in the cache before a new content hash list is added."); } DateTime expirationUtc = FingerprintTracker.GenerateNewExpiration(); var valueToAdd = new ContentHashListWithCacheMetadata( contentHashListWithDeterminism, expirationUtc, ContentAvailabilityGuarantee.NoContentBackedByCache); DateTime?rawExpiration = null; const int addLimit = 3; for (int addAttempts = 0; addAttempts < addLimit; addAttempts++) { var debugString = $"Adding contentHashList=[{valueToAdd.ContentHashListWithDeterminism.ContentHashList}] " + $"determinism=[{valueToAdd.ContentHashListWithDeterminism.Determinism}] to VSTS with " + $"contentAvailabilityGuarantee=[{valueToAdd.ContentGuarantee}], expirationUtc=[{expirationUtc}], forceUpdate=[{ForceUpdateOnAddContentHashList}]"; Tracer.Debug(context, debugString); Result <ContentHashListWithCacheMetadata> responseObject = await ContentHashListAdapter.AddContentHashListAsync( context, CacheNamespace, strongFingerprint, valueToAdd, forceUpdate: ForceUpdateOnAddContentHashList).ConfigureAwait(false); if (!responseObject.Succeeded) { return new AddOrGetContentHashListResult(responseObject); } ContentHashListWithCacheMetadata response = responseObject.Value; var inconsistencyErrorMessage = CheckForResponseInconsistency(response); if (inconsistencyErrorMessage != null) { return new AddOrGetContentHashListResult(inconsistencyErrorMessage); } rawExpiration = response.GetRawExpirationTimeUtc(); ContentHashList contentHashListToReturn = UnpackContentHashListAfterAdd(contentHashListWithDeterminism.ContentHashList, response); CacheDeterminism determinismToReturn = UnpackDeterminism(response, CacheId); bool needToUpdateExistingValue = await CheckNeedToUpdateExistingValueAsync( context, response, contentHashListToReturn, cts, urgencyHint).ConfigureAwait(false); if (!needToUpdateExistingValue) { SealIfNecessaryAfterUnbackedAddOrGet(context, strongFingerprint, contentHashListWithDeterminism, response); await TrackFingerprintAsync(context, strongFingerprint, rawExpiration).ConfigureAwait(false); return new AddOrGetContentHashListResult( new ContentHashListWithDeterminism(contentHashListToReturn, determinismToReturn)); } var hashOfExistingContentHashList = response.HashOfExistingContentHashList; Tracer.Debug(context, $"Attempting to replace unbacked value with hash {hashOfExistingContentHashList.ToHex()}"); valueToAdd = new ContentHashListWithCacheMetadata( contentHashListWithDeterminism, expirationUtc, ContentAvailabilityGuarantee.NoContentBackedByCache, hashOfExistingContentHashList ); } Tracer.Warning( context, $"Lost the AddOrUpdate race {addLimit} times against unbacked values. Returning as though the add succeeded for now."); await TrackFingerprintAsync(context, strongFingerprint, rawExpiration).ConfigureAwait(false); return new AddOrGetContentHashListResult(new ContentHashListWithDeterminism(null, CacheDeterminism.None)); }, traceOperationStarted: true, extraStartMessage: $"StrongFingerprint=({strongFingerprint}), ForceUpdate=({ForceUpdateOnAddContentHashList}) {contentHashListWithDeterminism.ToTraceString()}", extraEndMessage: _ => $"StrongFingerprint=({strongFingerprint}), ForceUpdate=({ForceUpdateOnAddContentHashList}) {contentHashListWithDeterminism.ToTraceString()}")); }