internal Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync(Context context, StrongFingerprint strongFingerprint, ContentHashListWithDeterminism contentHashListWithDeterminism, IContentSession contentSession, CancellationToken cts)
{
var ctx = new OperationContext(context, cts);
return(ctx.PerformOperationAsync(
_tracer,
async() =>
{
// We do multiple attempts here because we have a "CompareExchange" RocksDB in the heart
// of this implementation, and this may fail if the database is heavily contended.
// Unfortunately, there is not much we can do at the time of writing to avoid this
// requirement.
var maxAttempts = 5;
while (maxAttempts-- >= 0)
{
var contentHashList = contentHashListWithDeterminism.ContentHashList;
var determinism = contentHashListWithDeterminism.Determinism;
// Load old value. Notice that this get updates the time, regardless of whether we replace the value or not.
var oldContentHashListWithDeterminism = await _database.GetContentHashListAsync(ctx, strongFingerprint);
var(oldContentHashListInfo, replacementToken) = oldContentHashListWithDeterminism.Succeeded
? (oldContentHashListWithDeterminism.Value.contentHashListInfo, oldContentHashListWithDeterminism.Value.replacementToken)
: (default(ContentHashListWithDeterminism), string.Empty);
var oldContentHashList = oldContentHashListInfo.ContentHashList;
var oldDeterminism = oldContentHashListInfo.Determinism;
// Make sure we're not mixing SinglePhaseNonDeterminism records
if (!(oldContentHashList is null) && oldDeterminism.IsSinglePhaseNonDeterministic != determinism.IsSinglePhaseNonDeterministic)
{
return AddOrGetContentHashListResult.SinglePhaseMixingError;
}
if (oldContentHashList is null ||
oldDeterminism.ShouldBeReplacedWith(determinism) ||
!(await contentSession.EnsureContentIsAvailableAsync(ctx, oldContentHashList, ctx.Token).ConfigureAwait(false)))
{
// Replace if incoming has better determinism or some content for the existing
// entry is missing. The entry could have changed since we fetched the old value
// earlier, hence, we need to check it hasn't.
var exchanged = await _database.CompareExchange(
ctx,
strongFingerprint,
replacementToken,
oldContentHashListInfo,
contentHashListWithDeterminism).ThrowIfFailureAsync();
if (!exchanged)
{
// Our update lost, need to retry
continue;
}
return new AddOrGetContentHashListResult(new ContentHashListWithDeterminism(null, determinism));
}
// If we didn't accept the new value because it is the same as before, just with a not
// necessarily better determinism, then let the user know.
if (!(oldContentHashList is null) && oldContentHashList.Equals(contentHashList))
{
return new AddOrGetContentHashListResult(new ContentHashListWithDeterminism(null, oldDeterminism));
}
// If we didn't accept a deterministic tool's data, then we're in an inconsistent state
if (determinism.IsDeterministicTool)
{
return new AddOrGetContentHashListResult(
AddOrGetContentHashListResult.ResultCode.InvalidToolDeterminismError,
oldContentHashListWithDeterminism.Value.contentHashListInfo);
}
// If we did not accept the given value, return the value in the cache
return new AddOrGetContentHashListResult(oldContentHashListWithDeterminism);
}
return new AddOrGetContentHashListResult("Hit too many races attempting to add content hash list into the cache");
}));
/// <summary>
/// Store a ContentHashList
/// </summary>
internal Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync(
Context context,
StrongFingerprint strongFingerprint,
ContentHashListWithDeterminism contentHashListWithDeterminism,
IContentSession contentSession,
CancellationToken cts)
{
return(AddOrGetContentHashListCall.RunAsync(Tracer, new OperationContext(context, cts), strongFingerprint, async() =>
{
const int maxAttempts = 5;
int attemptCount = 0;
while (attemptCount++ <= maxAttempts)
{
var contentHashList = contentHashListWithDeterminism.ContentHashList;
var determinism = contentHashListWithDeterminism.Determinism;
// Load old value
var oldContentHashListWithDeterminism = await GetContentHashListAsync(strongFingerprint);
var oldContentHashList = oldContentHashListWithDeterminism.ContentHashList;
var oldDeterminism = oldContentHashListWithDeterminism.Determinism;
// Make sure we're not mixing SinglePhaseNonDeterminism records
if (oldContentHashList != null &&
(oldDeterminism.IsSinglePhaseNonDeterministic != determinism.IsSinglePhaseNonDeterministic))
{
return AddOrGetContentHashListResult.SinglePhaseMixingError;
}
// Match found.
// Replace if incoming has better determinism or some content for the existing entry is missing.
if (oldContentHashList == null || oldDeterminism.ShouldBeReplacedWith(determinism) ||
!(await contentSession.EnsureContentIsAvailableAsync(context, oldContentHashList, cts).ConfigureAwait(false)))
{
AddOrGetContentHashListResult contentHashListResult = await RunExclusiveAsync(
async() =>
{
// double check again.
var contentHashListInStore = await GetContentHashListAsync(strongFingerprint);
if (contentHashListInStore != oldContentHashListWithDeterminism)
{
return new AddOrGetContentHashListResult(contentHashListInStore);
}
var fileTimeUtc = _clock.UtcNow.ToFileTimeUtc();
await _replaceCommandPool.RunAsync(
new SQLiteParameter("@weakFingerprint", SerializeWithHashType(strongFingerprint.WeakFingerprint)),
new SQLiteParameter("@selectorContentHash", strongFingerprint.Selector.ContentHash.SerializeReverse()),
new SQLiteParameter("@selectorOutput", strongFingerprint.Selector.Output),
new SQLiteParameter("@fileTimeUtc", fileTimeUtc),
new SQLiteParameter("@payload", contentHashList.Payload?.ToArray()),
new SQLiteParameter("@determinism", determinism.EffectiveGuid.ToByteArray()),
new SQLiteParameter("@serializedDeterminism", determinism.Serialize()),
new SQLiteParameter("@contentHashList", contentHashList.Serialize()));
// Bump count if this is a new entry.
if (oldContentHashList == null)
{
IncrementCountOnAdd();
}
// Accept the value
return new AddOrGetContentHashListResult(
new ContentHashListWithDeterminism(null, determinism));
});
// our add lost - need to retry.
if (contentHashListResult.ContentHashListWithDeterminism.ContentHashList != null)
{
continue;
}
if (contentHashListResult.ContentHashListWithDeterminism.ContentHashList == null)
{
return contentHashListResult;
}
}
if (oldContentHashList != null && oldContentHashList.Equals(contentHashList))
{
return new AddOrGetContentHashListResult(
new ContentHashListWithDeterminism(null, oldDeterminism));
}
// If we didn't accept a deterministic tool's data, then we're in an inconsistent state
if (determinism.IsDeterministicTool)
{
return new AddOrGetContentHashListResult(
AddOrGetContentHashListResult.ResultCode.InvalidToolDeterminismError,
oldContentHashListWithDeterminism);
}
// If we did not accept the given value, return the value in the cache
return new AddOrGetContentHashListResult(oldContentHashListWithDeterminism);
}
return new AddOrGetContentHashListResult("Hit too many races attempting to add content hash list into the cache");
}));
}
/// <summary>
/// Store a ContentHashList
/// </summary>
internal async Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync(
Context context,
StrongFingerprint strongFingerprint,
ContentHashListWithDeterminism contentHashListWithDeterminism,
IContentSession contentSession,
CancellationToken cts)
{
using (var cancellableContext = TrackShutdown(context, cts))
{
return(await AddOrGetContentHashListCall.RunAsync(_tracer, cancellableContext, strongFingerprint, async() =>
{
using (await _lockSet.AcquireAsync(strongFingerprint.WeakFingerprint).ConfigureAwait(false))
{
var record = _records.FirstOrDefault(r => r.StrongFingerprint == strongFingerprint);
if (record == null)
{
// Match not found, add it.
record = new Record(strongFingerprint, contentHashListWithDeterminism);
_records.Add(record);
}
else
{
// Make sure we're not mixing SinglePhaseNonDeterminism records
if (record.ContentHashListWithDeterminism.Determinism.IsSinglePhaseNonDeterministic !=
contentHashListWithDeterminism.Determinism.IsSinglePhaseNonDeterministic)
{
return AddOrGetContentHashListResult.SinglePhaseMixingError;
}
// Match found.
// Replace if incoming has better determinism or some content for the existing entry is missing.
if (record.ContentHashListWithDeterminism.Determinism.ShouldBeReplacedWith(contentHashListWithDeterminism.Determinism) ||
!(await contentSession.EnsureContentIsAvailableAsync(context, record.ContentHashListWithDeterminism.ContentHashList, cts)
.ConfigureAwait(false)))
{
_records.Remove(record);
record = new Record(record.StrongFingerprint, contentHashListWithDeterminism);
_records.Add(record);
}
}
// Accept the value if it matches the final value in the cache
if (contentHashListWithDeterminism.ContentHashList.Equals(record.ContentHashListWithDeterminism.ContentHashList))
{
return new AddOrGetContentHashListResult(
new ContentHashListWithDeterminism(null, record.ContentHashListWithDeterminism.Determinism));
}
// If we didn't accept a deterministic tool's data, then we're in an inconsistent state
if (contentHashListWithDeterminism.Determinism.IsDeterministicTool)
{
return new AddOrGetContentHashListResult(
AddOrGetContentHashListResult.ResultCode.InvalidToolDeterminismError, record.ContentHashListWithDeterminism);
}
// If we did not accept the given value, return the value in the cache
return new AddOrGetContentHashListResult(record.ContentHashListWithDeterminism);
}
}));
}
}