internal Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync(Context context, StrongFingerprint strongFingerprint, ContentHashListWithDeterminism contentHashListWithDeterminism, IContentSession contentSession, CancellationToken cts) { var ctx = new OperationContext(context, cts); return(ctx.PerformOperationAsync( _tracer, async() => { // We do multiple attempts here because we have a "CompareExchange" RocksDB in the heart // of this implementation, and this may fail if the database is heavily contended. // Unfortunately, there is not much we can do at the time of writing to avoid this // requirement. var maxAttempts = 5; while (maxAttempts-- >= 0) { var contentHashList = contentHashListWithDeterminism.ContentHashList; var determinism = contentHashListWithDeterminism.Determinism; // Load old value. Notice that this get updates the time, regardless of whether we replace the value or not. var oldContentHashListWithDeterminism = await _database.GetContentHashListAsync(ctx, strongFingerprint); var(oldContentHashListInfo, replacementToken) = oldContentHashListWithDeterminism.Succeeded ? (oldContentHashListWithDeterminism.Value.contentHashListInfo, oldContentHashListWithDeterminism.Value.replacementToken) : (default(ContentHashListWithDeterminism), string.Empty); var oldContentHashList = oldContentHashListInfo.ContentHashList; var oldDeterminism = oldContentHashListInfo.Determinism; // Make sure we're not mixing SinglePhaseNonDeterminism records if (!(oldContentHashList is null) && oldDeterminism.IsSinglePhaseNonDeterministic != determinism.IsSinglePhaseNonDeterministic) { return AddOrGetContentHashListResult.SinglePhaseMixingError; } if (oldContentHashList is null || oldDeterminism.ShouldBeReplacedWith(determinism) || !(await contentSession.EnsureContentIsAvailableAsync(ctx, oldContentHashList, ctx.Token).ConfigureAwait(false))) { // Replace if incoming has better determinism or some content for the existing // entry is missing. The entry could have changed since we fetched the old value // earlier, hence, we need to check it hasn't. var exchanged = await _database.CompareExchange( ctx, strongFingerprint, replacementToken, oldContentHashListInfo, contentHashListWithDeterminism).ThrowIfFailureAsync(); if (!exchanged) { // Our update lost, need to retry continue; } return new AddOrGetContentHashListResult(new ContentHashListWithDeterminism(null, determinism)); } // If we didn't accept the new value because it is the same as before, just with a not // necessarily better determinism, then let the user know. if (!(oldContentHashList is null) && oldContentHashList.Equals(contentHashList)) { return new AddOrGetContentHashListResult(new ContentHashListWithDeterminism(null, oldDeterminism)); } // If we didn't accept a deterministic tool's data, then we're in an inconsistent state if (determinism.IsDeterministicTool) { return new AddOrGetContentHashListResult( AddOrGetContentHashListResult.ResultCode.InvalidToolDeterminismError, oldContentHashListWithDeterminism.Value.contentHashListInfo); } // If we did not accept the given value, return the value in the cache return new AddOrGetContentHashListResult(oldContentHashListWithDeterminism); } return new AddOrGetContentHashListResult("Hit too many races attempting to add content hash list into the cache"); }));
/// <summary> /// Store a ContentHashList /// </summary> internal Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync( Context context, StrongFingerprint strongFingerprint, ContentHashListWithDeterminism contentHashListWithDeterminism, IContentSession contentSession, CancellationToken cts) { return(AddOrGetContentHashListCall.RunAsync(Tracer, new OperationContext(context, cts), strongFingerprint, async() => { const int maxAttempts = 5; int attemptCount = 0; while (attemptCount++ <= maxAttempts) { var contentHashList = contentHashListWithDeterminism.ContentHashList; var determinism = contentHashListWithDeterminism.Determinism; // Load old value var oldContentHashListWithDeterminism = await GetContentHashListAsync(strongFingerprint); var oldContentHashList = oldContentHashListWithDeterminism.ContentHashList; var oldDeterminism = oldContentHashListWithDeterminism.Determinism; // Make sure we're not mixing SinglePhaseNonDeterminism records if (oldContentHashList != null && (oldDeterminism.IsSinglePhaseNonDeterministic != determinism.IsSinglePhaseNonDeterministic)) { return AddOrGetContentHashListResult.SinglePhaseMixingError; } // Match found. // Replace if incoming has better determinism or some content for the existing entry is missing. if (oldContentHashList == null || oldDeterminism.ShouldBeReplacedWith(determinism) || !(await contentSession.EnsureContentIsAvailableAsync(context, oldContentHashList, cts).ConfigureAwait(false))) { AddOrGetContentHashListResult contentHashListResult = await RunExclusiveAsync( async() => { // double check again. var contentHashListInStore = await GetContentHashListAsync(strongFingerprint); if (contentHashListInStore != oldContentHashListWithDeterminism) { return new AddOrGetContentHashListResult(contentHashListInStore); } var fileTimeUtc = _clock.UtcNow.ToFileTimeUtc(); await _replaceCommandPool.RunAsync( new SQLiteParameter("@weakFingerprint", SerializeWithHashType(strongFingerprint.WeakFingerprint)), new SQLiteParameter("@selectorContentHash", strongFingerprint.Selector.ContentHash.SerializeReverse()), new SQLiteParameter("@selectorOutput", strongFingerprint.Selector.Output), new SQLiteParameter("@fileTimeUtc", fileTimeUtc), new SQLiteParameter("@payload", contentHashList.Payload?.ToArray()), new SQLiteParameter("@determinism", determinism.EffectiveGuid.ToByteArray()), new SQLiteParameter("@serializedDeterminism", determinism.Serialize()), new SQLiteParameter("@contentHashList", contentHashList.Serialize())); // Bump count if this is a new entry. if (oldContentHashList == null) { IncrementCountOnAdd(); } // Accept the value return new AddOrGetContentHashListResult( new ContentHashListWithDeterminism(null, determinism)); }); // our add lost - need to retry. if (contentHashListResult.ContentHashListWithDeterminism.ContentHashList != null) { continue; } if (contentHashListResult.ContentHashListWithDeterminism.ContentHashList == null) { return contentHashListResult; } } if (oldContentHashList != null && oldContentHashList.Equals(contentHashList)) { return new AddOrGetContentHashListResult( new ContentHashListWithDeterminism(null, oldDeterminism)); } // If we didn't accept a deterministic tool's data, then we're in an inconsistent state if (determinism.IsDeterministicTool) { return new AddOrGetContentHashListResult( AddOrGetContentHashListResult.ResultCode.InvalidToolDeterminismError, oldContentHashListWithDeterminism); } // If we did not accept the given value, return the value in the cache return new AddOrGetContentHashListResult(oldContentHashListWithDeterminism); } return new AddOrGetContentHashListResult("Hit too many races attempting to add content hash list into the cache"); })); }
/// <summary> /// Store a ContentHashList /// </summary> internal async Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync( Context context, StrongFingerprint strongFingerprint, ContentHashListWithDeterminism contentHashListWithDeterminism, IContentSession contentSession, CancellationToken cts) { using (var cancellableContext = TrackShutdown(context, cts)) { return(await AddOrGetContentHashListCall.RunAsync(_tracer, cancellableContext, strongFingerprint, async() => { using (await _lockSet.AcquireAsync(strongFingerprint.WeakFingerprint).ConfigureAwait(false)) { var record = _records.FirstOrDefault(r => r.StrongFingerprint == strongFingerprint); if (record == null) { // Match not found, add it. record = new Record(strongFingerprint, contentHashListWithDeterminism); _records.Add(record); } else { // Make sure we're not mixing SinglePhaseNonDeterminism records if (record.ContentHashListWithDeterminism.Determinism.IsSinglePhaseNonDeterministic != contentHashListWithDeterminism.Determinism.IsSinglePhaseNonDeterministic) { return AddOrGetContentHashListResult.SinglePhaseMixingError; } // Match found. // Replace if incoming has better determinism or some content for the existing entry is missing. if (record.ContentHashListWithDeterminism.Determinism.ShouldBeReplacedWith(contentHashListWithDeterminism.Determinism) || !(await contentSession.EnsureContentIsAvailableAsync(context, record.ContentHashListWithDeterminism.ContentHashList, cts) .ConfigureAwait(false))) { _records.Remove(record); record = new Record(record.StrongFingerprint, contentHashListWithDeterminism); _records.Add(record); } } // Accept the value if it matches the final value in the cache if (contentHashListWithDeterminism.ContentHashList.Equals(record.ContentHashListWithDeterminism.ContentHashList)) { return new AddOrGetContentHashListResult( new ContentHashListWithDeterminism(null, record.ContentHashListWithDeterminism.Determinism)); } // If we didn't accept a deterministic tool's data, then we're in an inconsistent state if (contentHashListWithDeterminism.Determinism.IsDeterministicTool) { return new AddOrGetContentHashListResult( AddOrGetContentHashListResult.ResultCode.InvalidToolDeterminismError, record.ContentHashListWithDeterminism); } // If we did not accept the given value, return the value in the cache return new AddOrGetContentHashListResult(record.ContentHashListWithDeterminism); } })); } }