/// <summary>
/// Add or get content hash list for strong fingerprint
/// </summary>
public Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync(OperationContext context, StrongFingerprint strongFingerprint, ContentHashListWithDeterminism contentHashListWithDeterminism)
{
return(PerformOperationAsync(
context,
async sessionContext =>
{
var request = new AddOrGetContentHashListRequest
{
Header = sessionContext.CreateHeader(),
Fingerprint = strongFingerprint.ToGrpc(),
HashList = contentHashListWithDeterminism.ToGrpc(),
};
AddOrGetContentHashListResponse response = await SendGrpcRequestAndThrowIfFailedAsync(
sessionContext,
async() => await Client.AddOrGetContentHashListAsync(request),
throwFailures: false);
return response.FromGrpc();
}));
}
/// <summary>
/// Performs a compare exchange operation on metadata, while ensuring all invariants are kept. If the
/// fingerprint is not present, then it is inserted.
/// </summary>
public abstract Task <Result <bool> > CompareExchange(
OperationContext context,
StrongFingerprint strongFingerprint,
string expectedReplacementToken,
ContentHashListWithDeterminism expected,
ContentHashListWithDeterminism replacement);
/// <inheritdoc />
public override async Task <Result <(ContentHashListWithDeterminism contentHashListInfo, string replacementToken)> > GetContentHashListAsync(OperationContext context, StrongFingerprint strongFingerprint, bool preferShared)
{
var firstDatabase = preferShared ? _sharedDatabase : _localDatabase;
var secondDatabase = preferShared ? _localDatabase : _sharedDatabase;
var firstResult = await firstDatabase.GetContentHashListAsync(context, strongFingerprint, preferShared);
if (!firstResult.Succeeded || firstResult.Value.contentHashListInfo.ContentHashList != null)
{
return(firstResult);
}
return(await secondDatabase.GetContentHashListAsync(context, strongFingerprint, preferShared));
}
/// <inheritdoc />
public async Task <ObjectResult <ContentHashListWithCacheMetadata> > AddContentHashListAsync(
Context context,
string cacheNamespace,
StrongFingerprint strongFingerprint,
ContentHashListWithCacheMetadata valueToAdd)
{
try
{
Func <System.IO.Stream, System.Threading.CancellationToken, Task <StructResult <ContentHash> > > putStreamFunc =
async(stream, cts) =>
{
PutResult putResult = await _blobContentSession.PutStreamAsync(context, HashType.Vso0, stream, cts);
if (putResult.Succeeded)
{
return(new StructResult <ContentHash>(putResult.ContentHash));
}
return(new StructResult <ContentHash>(putResult));
};
StructResult <ContentHash> blobIdOfContentHashListResult =
await BlobContentHashListExtensions.PackInBlob(
putStreamFunc,
valueToAdd.ContentHashListWithDeterminism);
if (!blobIdOfContentHashListResult.Succeeded)
{
return(new ObjectResult <ContentHashListWithCacheMetadata>(blobIdOfContentHashListResult));
}
var blobContentHashListWithDeterminism =
new BlobContentHashListWithDeterminism(
valueToAdd.ContentHashListWithDeterminism.Determinism.EffectiveGuid,
BuildXL.Cache.ContentStore.Hashing.BlobIdentifierHelperExtensions.ToBlobIdentifier(blobIdOfContentHashListResult.Data));
var blobContentHashListWithCacheMetadata = new BlobContentHashListWithCacheMetadata(
blobContentHashListWithDeterminism,
valueToAdd.GetRawExpirationTimeUtc(),
valueToAdd.ContentGuarantee,
valueToAdd.HashOfExistingContentHashList);
BlobContentHashListResponse addResult = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync(
context,
"AddContentHashList",
innerCts => _buildCacheHttpClient.AddContentHashListAsync(
cacheNamespace,
strongFingerprint,
blobContentHashListWithCacheMetadata),
CancellationToken.None).ConfigureAwait(false);
DownloadUriCache.Instance.BulkAddDownloadUris(addResult.BlobDownloadUris);
// add succeeded but returned an empty contenthashlistwith cache metadata. correct this.
if (addResult.ContentHashListWithCacheMetadata == null)
{
return
(new ObjectResult <ContentHashListWithCacheMetadata>(
new ContentHashListWithCacheMetadata(
new ContentHashListWithDeterminism(null, blobContentHashListWithCacheMetadata.Determinism),
blobContentHashListWithCacheMetadata.GetRawExpirationTimeUtc(),
blobContentHashListWithCacheMetadata.ContentGuarantee)));
}
else
{
return(await UnpackBlobContentHashListAsync(context, addResult.ContentHashListWithCacheMetadata));
}
}
catch (Exception ex)
{
return(new ObjectResult <ContentHashListWithCacheMetadata>(ex));
}
}
/// <summary>
/// Initializes a new instance of the <see cref="AddOrGetContentHashListCall"/> class.
/// </summary>
private AddOrGetContentHashListCall(MemoizationStoreTracer tracer, OperationContext context, StrongFingerprint fingerprint)
: base(tracer, context)
{
Tracer.AddOrGetContentHashListStart(Context, fingerprint);
}
/// <summary>
/// Takes a strong fingerprint and returns the deserialized contents of
/// the input assertion list file that corresponds to it
/// </summary>
/// <param name="sfp">The strong fingerprint to get the input assertion
/// list file contents for</param>
/// <param name="cacheErrors">Any cache errors that are found will be
/// added to this collection</param>
/// <returns>Deserialized contents of the input assertion list file
/// corresponding to the specified strong fingerprint</returns>
private async Task <string> GetInputAssertionListFileContentsAsync(StrongFingerprint sfp, ConcurrentDictionary <CacheError, int> cacheErrors)
{
// Check for the NoItem
if (sfp.CasElement.Equals(CasHash.NoItem))
{
return(string.Empty);
}
// Pin the input assertion list file
Possible <string, Failure> possibleString = await m_readOnlySession.PinToCasAsync(sfp.CasElement).ConfigureAwait(false);
if (!possibleString.Succeeded)
{
return(string.Empty);
}
// Get the stream for the input assertion list file
Possible <StreamWithLength, Failure> possibleStream = await m_readOnlySession.GetStreamAsync(sfp.CasElement).ConfigureAwait(false);
if (!possibleStream.Succeeded)
{
cacheErrors.TryAdd(
new CacheError(
CacheErrorType.CasHashError,
"The input assertion list for SFP " + sfp.ToString() + " was not found in CAS"), 0);
return(string.Empty);
}
// Read the stream contents while hashing
return(await Task.Run(() =>
{
using (var hasher = ContentHashingUtilities.HashInfo.CreateContentHasher())
{
using (var hashingStream = hasher.CreateReadHashingStream(possibleStream.Result))
{
using (var reader = new BuildXLReader(false, hashingStream, false))
{
var maybePathSet = ObservedPathSet.TryDeserialize(s_pathTable, reader);
// Check if deserialization was successful
if (!maybePathSet.Succeeded)
{
// Deserialization failed
cacheErrors.TryAdd(
new CacheError(
CacheErrorType.CasHashError,
"The input assertion list for SFP " + sfp.ToString() + " could not be deserialized"), 0);
return string.Empty;
}
CasHash newCasHash = new CasHash(hashingStream.GetContentHash());
// Check if the hashes match
if (!sfp.CasElement.Equals(newCasHash))
{
cacheErrors.TryAdd(
new CacheError(
CacheErrorType.CasHashError,
"The input assertion list for SFP " + sfp.ToString() + " has been altered in the CAS"), 0);
return string.Empty;
}
// Deserialization was successful and file was unaltered
StringBuilder fileContents = new StringBuilder();
foreach (ObservedPathEntry entry in maybePathSet.Result.Paths)
{
fileContents.Append(entry.Path.ToString(s_pathTable)).Append(Environment.NewLine);
}
return fileContents.ToString();
}
}
}
}).ConfigureAwait(false));
}
/// <nodoc /> protected abstract Task <ContentHashListResult> GetContentHashListCoreAsync(OperationContext context, StrongFingerprint strongFingerprint, bool preferShared);
private Task AssertExpirationInRangeAsync(Context context, string cacheNamespace, StrongFingerprint strongFingerprint, DateTime expectedLow, DateTime expectedHigh)
{
// Create a bare BuildCache client so that the value read is not thwarted by some intermediate cache layer (like Redis)
Func <DisposableDirectory, ICache> createCheckerCacheFunc =
dir => CreateBareBuildCache(dir, cacheNamespace, FileSystem, Logger, BackingOption.WriteThrough, ItemStorageOption);
Func <ICacheSession, Task> checkFunc = async(ICacheSession checkSession) =>
{
// Bare BuildCache clients should only produce BuildCacheSessions
BuildCacheSession buildCacheSession = checkSession as BuildCacheSession;
Assert.NotNull(buildCacheSession);
// Raw expiration is only visible to the adapter, not through the ICacheSession APIs.
// This also has the nice (non-)side-effect of not updating the expiration as part of *this* read.
IContentHashListAdapter buildCacheHttpClientAdapter = buildCacheSession.ContentHashListAdapter;
ObjectResult <ContentHashListWithCacheMetadata> getResult = await buildCacheHttpClientAdapter.GetContentHashListAsync(context, cacheNamespace, strongFingerprint);
Assert.True(getResult.Succeeded);
Assert.NotNull(getResult.Data);
DateTime?rawExpiration = getResult.Data.GetRawExpirationTimeUtc();
Assert.NotNull(rawExpiration);
// Verify that the raw expiration (i.e. the value's TTL w/o consideration for content existence or whether the value might be replaced) is visible and correct
TimeSpan assertionTolerance = TimeSpan.FromHours(1); // The test time + the simulator's clock skew hopefully won't exceed 1 hour.
Assert.InRange(rawExpiration.Value, expectedLow - assertionTolerance, expectedHigh + assertionTolerance);
};
return(RunTestAsync(context, checkFunc, createCheckerCacheFunc));
}
public async Task <Possible <FullCacheRecordWithDeterminism, Failure> > AddOrGetAsync(WeakFingerprintHash weak, CasHash casElement, Hash hashElement, CasEntries hashes, UrgencyHint urgencyHint, Guid activityId)
{
Contract.Requires(!IsClosed);
Contract.Requires(hashes.IsValid);
Contract.Assert(!IsReadOnly);
using (var counter = m_counters.AddOrGetCounter())
{
using (var eventing = new AddOrGetActivity(BasicFilesystemCache.EventSource, activityId, this))
{
eventing.Start(weak, casElement, hashElement, hashes, urgencyHint);
counter.SetEntriesCount(hashes.Count); // The size of what we are adding (effectively)
// We check the Cas entries if we are strict
if (StrictMetadataCasCoupling)
{
// Check that the content is valid.
if (!m_pinnedToCas.ContainsKey(casElement))
{
counter.Failed();
return(eventing.StopFailure(new UnpinnedCasEntryFailure(CacheId, casElement)));
}
foreach (CasHash hash in hashes)
{
if (!m_pinnedToCas.ContainsKey(hash))
{
counter.Failed();
return(eventing.StopFailure(new UnpinnedCasEntryFailure(CacheId, hash)));
}
}
}
StrongFingerprint strong = new StrongFingerprint(weak, casElement, hashElement, CacheId);
// Assume we accepted the Add and there is nothing to return
FullCacheRecord result = null;
string strongFingerprintName = m_cache.GetStrongFingerprintFilename(strong);
try
{
using (FileStream file = await m_cache.ContendedOpenStreamAsync(strongFingerprintName, FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
// The compiler thinks that it is not assigned down below at the end
// even though it would be set by the writeEntries being true in that case
CasEntries oldCasEntries = hashes;
// Assume we will write our new enties to the file.
bool writeEntries = true;
// If there is some data in the file already, we need to try to read it.
if (file.Length > 0)
{
var possibleOldCasEntries = await m_cache.ReadCacheEntryAsync(file);
// Only if it was formatted correctly do we continue to check if
// we should replace it.
if (possibleOldCasEntries.Succeeded)
{
oldCasEntries = possibleOldCasEntries.Result;
writeEntries = false;
// We can only replace if both or neither is SinglePhaseNonDeterministic
if (oldCasEntries.Determinism.IsSinglePhaseNonDeterministic != hashes.Determinism.IsSinglePhaseNonDeterministic)
{
counter.Failed();
return(eventing.StopFailure(new SinglePhaseMixingFailure(CacheId)));
}
// Should we replace?
if (hashes.Determinism.IsSinglePhaseNonDeterministic ||
(!oldCasEntries.Determinism.IsDeterministicTool &&
(hashes.Determinism.IsDeterministic && !oldCasEntries.Determinism.Equals(hashes.Determinism))))
{
// We are replacing due to determinism
counter.Det();
writeEntries = true;
}
else if (HasMissingContent(oldCasEntries))
{
counter.Repair();
writeEntries = true;
}
else if (oldCasEntries.Determinism.IsDeterministicTool && hashes.Determinism.IsDeterministicTool && !oldCasEntries.Equals(hashes))
{
// We have a non-deterministic tool!
counter.Failed();
return(eventing.StopFailure(new NotDeterministicFailure(CacheId, new FullCacheRecord(strong, oldCasEntries), new FullCacheRecord(strong, hashes))));
}
}
}
// Are we going to write the entries?
if (writeEntries)
{
// We are writing so the old entries don't count
oldCasEntries = hashes;
// Write from the front
file.SetLength(0);
await m_cache.WriteCacheEntryAsync(file, hashes);
}
else
{
counter.Dup();
}
// If what is in the cache is different than what we are
// asking to add, build a FullCacheRecord to return what
// is in the cache.
if (!oldCasEntries.Equals(hashes))
{
counter.Get();
result = new FullCacheRecord(strong, oldCasEntries);
}
}
}
catch (Exception e)
{
counter.Failed();
return(eventing.StopFailure(new StrongFingerprintAccessFailure(m_cache.CacheId, strong, e)));
}
AddSessionRecord(strong);
if (result != null)
{
return
(eventing.Returns(
new FullCacheRecordWithDeterminism(
new FullCacheRecord(
result.StrongFingerprint,
result.CasEntries.GetModifiedCasEntriesWithDeterminism(
m_cache.IsAuthoritative,
m_cache.CacheGuid,
CacheDeterminism.NeverExpires)))));
}
else
{
return(eventing.Returns(new FullCacheRecordWithDeterminism(hashes.GetFinalDeterminism(m_cache.IsAuthoritative, m_cache.CacheGuid, DateTime.UtcNow.Add(m_cache.TimeToLive)))));
}
}
}
}
/// <inheritdoc />
public async Task <GetContentHashListResult> GetOrAddContentHashListAsync(Context context, StrongFingerprint strongFingerprint, Func <StrongFingerprint, Task <GetContentHashListResult> > getFuncAsync)
{
try
{
var cacheKey = _redisSerializer.ToRedisKey(strongFingerprint);
RedisValue cacheResult;
Stopwatch stopwatch = Stopwatch.StartNew();
try
{
_cacheTracer.GetContentHashListStart(context);
cacheResult = await StringGetWithExpiryBumpAsync(context, cacheKey);
}
finally
{
_cacheTracer.GetContentHashListStop(context, stopwatch.Elapsed);
}
if (!cacheResult.HasValue)
{
_cacheTracer.RecordContentHashListFetchedFromBackingStore(context, strongFingerprint);
GetContentHashListResult getResult = await getFuncAsync(strongFingerprint);
if (getResult.Succeeded)
{
var cacheValue = _redisSerializer.ToRedisValue(getResult.ContentHashListWithDeterminism);
stopwatch = Stopwatch.StartNew();
try
{
_cacheTracer.AddContentHashListStart(context);
bool result = await StringSetWithExpiryBumpAsync(context, cacheKey, cacheValue);
context.Debug($"Added redis cache entry for {strongFingerprint}: {getResult}. Result: {result}");
}
finally
{
_cacheTracer.AddContentHashListStop(context, stopwatch.Elapsed);
}
}
return(getResult);
}
else
{
_cacheTracer.RecordGetContentHashListFetchedDistributed(context, strongFingerprint);
}
return(new GetContentHashListResult(_redisSerializer.AsContentHashList(cacheResult)));
}
catch (Exception ex)
{
return(new GetContentHashListResult(ex));
}
}
/// <inheritdoc />
public Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync(Context context, StrongFingerprint strongFingerprint, ContentHashListWithDeterminism contentHashListWithDeterminism, CancellationToken cts, UrgencyHint urgencyHint)
{
return(WithOperationContext(
context,
cts,
operationContext =>
{
return operationContext.PerformOperationAsync(
DistributedTracer,
async() =>
{
// Metadata cache assumes no guarantees about the fingerprints added, hence invalidate the cache and serve the request using backing store
var cacheResult = await MetadataCache.DeleteFingerprintAsync(context, strongFingerprint);
if (!cacheResult.Succeeded)
{
Tracer.Error(context, $"Error while removing fingerprint {strongFingerprint} from metadata cache. Result: {cacheResult}.");
}
return await _innerCacheSession.AddOrGetContentHashListAsync(context, strongFingerprint, contentHashListWithDeterminism, cts, urgencyHint);
},
traceOperationStarted: true,
extraStartMessage: $"StrongFingerprint=({strongFingerprint}) {contentHashListWithDeterminism.ToTraceString()}",
extraEndMessage: _ => $"StrongFingerprint=({strongFingerprint}) {contentHashListWithDeterminism.ToTraceString()}");
}));
}
public async Task SimpleDummySession()
{
const string TestName = "SimpleSession";
string testCacheId = MakeCacheId(TestName);
ICache cache = await CreateCacheAsync(testCacheId);
// Now for the session (which we base on the cache ID)
string testSessionId = "Session1-" + testCacheId;
ICacheSession session = await CreateSessionAsync(cache, testSessionId);
// Do the default fake build for this test (first time, no cache hit)
FullCacheRecord built = await FakeBuild.DoPipAsync(session, TestName);
XAssert.AreEqual(FakeBuild.NewRecordCacheId, built.CacheId, "Should have been a new cache entry!");
// Now we see if we can get back the items we think we should
await CloseSessionAsync(session, testSessionId);
// We need a read only session to get the CasEntries
ICacheReadOnlySession readOnlySession = (await cache.CreateReadOnlySessionAsync()).Success();
// Validate that the cache contains a dummy session and it has the one cache record it needs.
HashSet <FullCacheRecord> found = new HashSet <FullCacheRecord>();
foreach (var strongFingerprintTask in cache.EnumerateSessionStrongFingerprints(MemoizationStoreAdapterCache.DummySessionName).Success().OutOfOrderTasks())
{
StrongFingerprint strongFingerprint = await strongFingerprintTask;
CasEntries casEntries = (await readOnlySession.GetCacheEntryAsync(strongFingerprint)).Success();
FullCacheRecord record = new FullCacheRecord(strongFingerprint, casEntries);
// If it is not the record we already found...
if (!found.Contains(record))
{
found.Add(record);
}
XAssert.AreEqual(1, found.Count, "There should be only 1 unique record in the session");
XAssert.AreEqual(built.StrongFingerprint.WeakFingerprint, record.StrongFingerprint.WeakFingerprint);
XAssert.AreEqual(built.StrongFingerprint.CasElement, record.StrongFingerprint.CasElement);
XAssert.AreEqual(built.StrongFingerprint.HashElement, record.StrongFingerprint.HashElement);
XAssert.AreEqual(built.CasEntries.Count, record.CasEntries.Count, "Did not return the same number of items");
XAssert.IsTrue(record.CasEntries.Equals(built.CasEntries), "Items returned are not the same hash and/or order order");
XAssert.AreEqual(built, record);
// We can not check record.CasEntries.IsDeterministic
// as the cache may have determined that they are deterministic
// via cache determinism recovery.
}
XAssert.AreEqual(1, found.Count, "There should be 1 and only 1 record in the session!");
await readOnlySession.CloseAsync().SuccessAsync();
// Check that the cache has the items in it
await FakeBuild.CheckContentsAsync(cache, built);
// Now redo the "build" with a cache hit
testSessionId = "Session2-" + testCacheId;
session = await CreateSessionAsync(cache, testSessionId);
FullCacheRecord rebuilt = await FakeBuild.DoPipAsync(session, TestName);
XAssert.AreEqual(built, rebuilt, "Should have been the same build!");
// We make sure we did get it from a cache rather than a manual rebuild.
XAssert.AreNotEqual(built.CacheId, rebuilt.CacheId, "Should not be the same cache ID");
await CloseSessionAsync(session, testSessionId);
readOnlySession = await cache.CreateReadOnlySessionAsync().SuccessAsync();
// Now that we have done the second build via a cache hit, it should produce the
// same cache record as before
foreach (var strongFingerprintTask in cache.EnumerateSessionStrongFingerprints(MemoizationStoreAdapterCache.DummySessionName).Success().OutOfOrderTasks())
{
StrongFingerprint strongFingerprint = await strongFingerprintTask;
CasEntries casEntries = (await readOnlySession.GetCacheEntryAsync(strongFingerprint)).Success();
FullCacheRecord record = new FullCacheRecord(strongFingerprint, casEntries);
XAssert.IsTrue(found.Contains(record), "Second session should produce the same cache record but did not!");
}
(await readOnlySession.CloseAsync()).Success();
await ShutdownCacheAsync(cache, testCacheId);
}
public Task EvictionInLruOrder()
{
var context = new Context(Logger);
return(RunTestAsync(context, async session =>
{
// Write more than MaxRowCount items so the first ones should fall out.
var strongFingerprints = Enumerable.Range(0, (int)MaxRowCount + 3).Select(i => StrongFingerprint.Random()).ToList();
foreach (var strongFingerprint in strongFingerprints)
{
await session.AddOrGetContentHashListAsync(
context,
strongFingerprint,
new ContentHashListWithDeterminism(ContentHashList.Random(), CacheDeterminism.None),
Token).ShouldBeSuccess();
_clock.Increment();
}
// Make sure store purging completes.
await((ReadOnlySQLiteMemoizationSession)session).PurgeAsync(context);
// Check the first items written have fallen out.
for (var i = 0; i < 3; i++)
{
GetContentHashListResult r = await session.GetContentHashListAsync(context, strongFingerprints[i], Token);
r.Succeeded.Should().BeTrue();
r.ContentHashListWithDeterminism.ContentHashList.Should().BeNull();
}
// Check the rest are still present.
for (var i = 3; i < strongFingerprints.Count; i++)
{
GetContentHashListResult r = await session.GetContentHashListAsync(context, strongFingerprints[i], Token);
r.Succeeded.Should().BeTrue();
r.ContentHashListWithDeterminism.ContentHashList.Should().NotBeNull();
}
}));
}
/// <summary>
/// Get content hash list for strong fingerprint
/// </summary>
public Task <GetContentHashListResult> GetContentHashListAsync(OperationContext context, StrongFingerprint strongFingerprint)
{
return(PerformOperationAsync(
context,
async sessionContext =>
{
var request = new GetContentHashListRequest()
{
Header = sessionContext.CreateHeader(),
Fingerprint = strongFingerprint.ToGrpc(),
};
GetContentHashListResponse response = await SendGrpcRequestAndThrowIfFailedAsync(
sessionContext,
async() => await Client.GetContentHashListAsync(request));
return response.FromGrpc();
}));
}
/// <inheritdoc />
public override Task <Result <bool> > CompareExchange(OperationContext context, StrongFingerprint strongFingerprint, string replacementToken, ContentHashListWithDeterminism expected, ContentHashListWithDeterminism replacement)
{
return(Task.FromResult(Database.CompareExchange(context, strongFingerprint, expected, replacement).ToResult()));
}
public Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync(Context context, StrongFingerprint strongFingerprint, ContentHashListWithDeterminism contentHashListWithDeterminism, CancellationToken cts, UrgencyHint urgencyHint = UrgencyHint.Nominal)
{
AddOrGetContentHashListAsyncParams.Add(new Tuple <StrongFingerprint, ContentHashListWithDeterminism>(strongFingerprint, contentHashListWithDeterminism));
return(Task.FromResult(new AddOrGetContentHashListResult(contentHashListWithDeterminism)));
}
/// <inheritdoc />
public override Task <Result <(ContentHashListWithDeterminism contentHashListInfo, string replacementToken)> > GetContentHashListAsync(OperationContext context, StrongFingerprint strongFingerprint, bool preferShared)
{
var contentHashListResult = Database.GetContentHashList(context, strongFingerprint);
return(contentHashListResult.Succeeded
? Task.FromResult(new Result <(ContentHashListWithDeterminism, string)>((contentHashListResult.ContentHashListWithDeterminism, string.Empty)))
: Task.FromResult(new Result <(ContentHashListWithDeterminism, string)>(contentHashListResult)));
}
public Task <GetContentHashListResult> GetContentHashListAsync(Context context, StrongFingerprint strongFingerprint, CancellationToken cts, UrgencyHint urgencyHint = UrgencyHint.Nominal)
{
GetContentHashListAsyncParams.Add(strongFingerprint);
return(Task.FromResult(new GetContentHashListResult(new ContentHashListWithDeterminism())));
}
/// <summary>
/// Load a ContentHashList and the token used to replace it.
/// </summary>
public Task <ContentHashListResult> GetContentHashListAsync(OperationContext context, StrongFingerprint strongFingerprint, bool preferShared)
{
return(context.PerformOperationWithTimeoutAsync(
Tracer,
nestedContext => GetContentHashListCoreAsync(nestedContext, strongFingerprint, preferShared),
extraStartMessage: $"StrongFingerprint=[{strongFingerprint}], PreferShared=[{preferShared}]",
extraEndMessage: result => getStringResult(result),
timeout: _timeout));
string getStringResult(ContentHashListResult result)
{
var resultString = $"StrongFingerprint=[{strongFingerprint}], PreferShared=[{preferShared}] Result=[{result.GetValueOrDefault().contentHashListInfo.ToTraceString()}] Token=[{result.GetValueOrDefault().replacementToken}]";
if (result.Source != ContentHashListSource.Unknown)
{
resultString += $", Source=[{result.Source}]";
}
return(resultString);
}
}
/// <inheritdoc />
public Task <GetContentHashListResult> GetContentHashListAsync(Context context, StrongFingerprint strongFingerprint, CancellationToken cts, UrgencyHint urgencyHint)
{
return(GetContentHashListCall.RunAsync(DistributedTracer, context, strongFingerprint, async() =>
{
GetContentHashListResult innerResult = null;
// Get the value from the metadata cache or load the current inner value into it (and then return it)
var existing = await MetadataCache.GetOrAddContentHashListAsync(context, strongFingerprint, async fingerprint =>
{
innerResult = await _innerCacheSession.GetContentHashListAsync(context, fingerprint, cts, urgencyHint);
return innerResult;
});
// Check to see if we need to need to read through to the inner value.
if (_readThroughMode == ReadThroughMode.ReadThrough &&
existing.Succeeded &&
!(existing.ContentHashListWithDeterminism.Determinism.IsDeterministic &&
existing.ContentHashListWithDeterminism.Determinism.Guid == _innerCacheId))
{
// Read through to the inner cache because the metadata cache's value is not guaranteed to be backed.
if (innerResult == null)
{
// We did not already query the inner cache as part of the original query, so do that now.
innerResult = await _innerCacheSession.GetContentHashListAsync(context, strongFingerprint, cts, urgencyHint);
}
if (innerResult != null && innerResult.Succeeded &&
innerResult.ContentHashListWithDeterminism.Determinism.IsDeterministic)
{
// If the inner cache's value is now backed, clear the value from the metadata cache so that the
// next read will load the backed value into the metadata cache (preventing the need for future read-throughs).
await MetadataCache.DeleteFingerprintAsync(context, strongFingerprint).IgnoreFailure();
}
return innerResult;
}
else
{
// Return the existing value in the metadata cache, or any error.
return existing;
}
}));
}
public Task BasicDistributedAddAndGet()
{
_test.ConfigureWithOneMaster(dcs =>
{
dcs.TouchContentHashLists = true;
});
return(RunTestAsync(
3,
async context =>
{
var sf = StrongFingerprint.Random();
var touchedSf = StrongFingerprint.Random();
var workerCaches = context.EnumerateWorkersIndices().Select(i => context.CacheSessions[i]).ToArray();
var workerCache0 = workerCaches[0];
var workerCache1 = workerCaches[1];
var masterCache = context.CacheSessions[context.GetMasterIndex()];
var workerStores = context.EnumerateWorkersIndices().Select(i => context.GetLocalLocationStore(i)).ToArray();
var workerStore0 = workerStores[0];
var workerStore1 = workerStores[1];
var masterStore = context.GetLocalLocationStore(context.GetMasterIndex());
var putResult = await workerCache0.PutRandomAsync(
context,
ContentHashType,
false,
RandomContentByteCount,
Token).ShouldBeSuccess();
var contentHashList = new ContentHashList(new[] { putResult.ContentHash });
async Task <int?> findLevelAsync(ICacheSession cacheSession)
{
var levelCacheSession = (ILevelSelectorsProvider)cacheSession;
int level = 0;
while (true)
{
// Only levels 1 and 2 should be available from cache session
level.Should().BeLessThan(2);
var result = await levelCacheSession.GetLevelSelectorsAsync(context, sf.WeakFingerprint, Token, level)
.ShouldBeSuccess();
if (result.Value.Selectors.Any(s => s.Equals(sf.Selector)))
{
return level;
}
else if (!result.Value.HasMore)
{
return null;
}
level++;
}
}
async Task ensureLevelAsync(ICacheSession cacheSession, int?expectedLevel)
{
if (expectedLevel != null)
{
var getResult = await cacheSession.GetContentHashListAsync(context, sf, Token).ShouldBeSuccess();
Assert.Equal(contentHashList, getResult.ContentHashListWithDeterminism.ContentHashList);
}
var level = await findLevelAsync(cacheSession);
level.Should().Be(expectedLevel);
}
DateTime getTouchedFingerprintLastAccessTime(LocalLocationStore store)
{
return store.Database.GetMetadataEntry(context, touchedSf, touch: false).ShouldBeSuccess().Value.Value.LastAccessTimeUtc;
}
// Verify not found initially
await ensureLevelAsync(workerCache0, null);
await ensureLevelAsync(workerCache1, null);
var addResult = await workerCache0.AddOrGetContentHashListAsync(
context,
sf,
new ContentHashListWithDeterminism(contentHashList, CacheDeterminism.None),
Token).ShouldBeSuccess();
await workerCache0.AddOrGetContentHashListAsync(
context,
touchedSf,
new ContentHashListWithDeterminism(contentHashList, CacheDeterminism.None),
Token).ShouldBeSuccess();
Assert.Equal(null, addResult.ContentHashListWithDeterminism.ContentHashList);
await ensureLevelAsync(masterCache, 0 /* Master DB gets updated immediately */);
// Verify found remotely
await ensureLevelAsync(workerCache0, 1);
await ensureLevelAsync(workerCache1, 1);
// Get original last access time for strong fingerprint which will be touched
var initialTouchTime = getTouchedFingerprintLastAccessTime(masterStore);
initialTouchTime.Should().Be(_test.TestClock.UtcNow);
_test.TestClock.UtcNow += TimeSpan.FromDays(1);
// Restore (update) db on worker 1
await masterStore.CreateCheckpointAsync(context).ShouldBeSuccess();
await workerStore1.HeartbeatAsync(context, inline: true, forceRestore: true).ShouldBeSuccess();
await ensureLevelAsync(workerCache0, 1 /* Worker 0 has not updated its db it should still be found remotely */);
await ensureLevelAsync(workerCache1, 0 /* Worker 1 should find locally after restoring DB */);
// Replace entry
putResult = await workerCache0.PutRandomAsync(
context,
ContentHashType,
false,
RandomContentByteCount,
Token).ShouldBeSuccess();
contentHashList = new ContentHashList(new[] { putResult.ContentHash });
await workerCache0.AddOrGetContentHashListAsync(
context,
sf,
new ContentHashListWithDeterminism(contentHashList, CacheDeterminism.None),
Token).ShouldBeSuccess();
await workerCache0.GetContentHashListAsync(
context,
touchedSf,
Token).ShouldBeSuccess();
await ensureLevelAsync(masterCache, 0 /* Master db gets updated immediately */);
await masterStore.CreateCheckpointAsync(context).ShouldBeSuccess();
await workerStore1.HeartbeatAsync(context, inline: true, forceRestore: true).ShouldBeSuccess();
await ensureLevelAsync(workerCache0, 1 /* Worker 0 has not updated its db it should still be found remotely */);
await ensureLevelAsync(workerCache1, 0 /* Worker 1 should find locally after restoring DB */);
// Touch should have propagated to master and worker 1 (worker 1 restored checkpoint above)
getTouchedFingerprintLastAccessTime(masterStore).Should().Be(_test.TestClock.UtcNow);
getTouchedFingerprintLastAccessTime(workerStore1).Should().Be(_test.TestClock.UtcNow);
}));
}
/// <inheritdoc />
public Task <GetContentHashListResult> GetContentHashListAsync(Context context, StrongFingerprint strongFingerprint, CancellationToken cts, UrgencyHint urgencyHint = UrgencyHint.Nominal)
{
return(MemoizationStore.GetContentHashListAsync(context, strongFingerprint, cts));
}
/// <inheritdoc />
public async Task <ObjectResult <ContentHashListWithCacheMetadata> > GetContentHashListAsync(Context context, string cacheNamespace, StrongFingerprint strongFingerprint)
{
try
{
if (!BlobContentHashListCache.Instance.TryGetValue(cacheNamespace, strongFingerprint, out var blobCacheMetadata))
{
BlobContentHashListResponse blobResponse = await ArtifactHttpClientErrorDetectionStrategy.ExecuteWithTimeoutAsync(
context,
"GetContentHashList",
innerCts => _buildCacheHttpClient.GetContentHashListAsync(cacheNamespace, strongFingerprint),
CancellationToken.None)
.ConfigureAwait(false);
blobCacheMetadata = blobResponse.ContentHashListWithCacheMetadata;
DownloadUriCache.Instance.BulkAddDownloadUris(blobResponse.BlobDownloadUris);
AddDownloadUriToCache(blobCacheMetadata.ContentHashListWithDeterminism);
}
// Currently expect the Blob-based service to return null on misses,
// but the other catches have been left for safety/compat.
if (blobCacheMetadata == null)
{
return(new ObjectResult <ContentHashListWithCacheMetadata>(EmptyContentHashList));
}
return(await UnpackBlobContentHashListAsync(context, blobCacheMetadata));
}
catch (ContentBagNotFoundException)
{
return(new ObjectResult <ContentHashListWithCacheMetadata>(EmptyContentHashList));
}
catch (CacheServiceException ex) when(ex.ReasonCode == CacheErrorReasonCode.ContentHashListNotFound)
{
return(new ObjectResult <ContentHashListWithCacheMetadata>(EmptyContentHashList));
}
catch (VssServiceResponseException serviceEx) when(serviceEx.HttpStatusCode == HttpStatusCode.NotFound)
{
return(new ObjectResult <ContentHashListWithCacheMetadata>(EmptyContentHashList));
}
catch (Exception ex)
{
return(new ObjectResult <ContentHashListWithCacheMetadata>(ex));
}
}
/// <inheritdoc />
public override GetContentHashListResult GetContentHashList(OperationContext context, StrongFingerprint strongFingerprint)
{
throw new NotImplementedException();
}
/// <inheritdoc />
public Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync(Context context, StrongFingerprint strongFingerprint, ContentHashListWithDeterminism contentHashListWithDeterminism, CancellationToken cts, UrgencyHint urgencyHint)
{
return(AddOrGetContentHashListCall.RunAsync(DistributedTracer, OperationContext(context), strongFingerprint, async() =>
{
// Metadata cache assumes no guarantees about the fingerprints added, hence invalidate the cache and serve the request using backing store
var cacheResult = await MetadataCache.DeleteFingerprintAsync(context, strongFingerprint);
if (!cacheResult.Succeeded)
{
context.Error($"Error while removing fingerprint {strongFingerprint} from metadata cache. Result: {cacheResult}.");
}
return await _innerCacheSession.AddOrGetContentHashListAsync(context, strongFingerprint, contentHashListWithDeterminism, cts, urgencyHint);
}));
}
/// <summary>
/// Store a ContentHashList
/// </summary>
internal async Task <AddOrGetContentHashListResult> AddOrGetContentHashListAsync(
Context context,
StrongFingerprint strongFingerprint,
ContentHashListWithDeterminism contentHashListWithDeterminism,
IContentSession contentSession,
CancellationToken cts)
{
using (var cancellableContext = TrackShutdown(context, cts))
{
return(await AddOrGetContentHashListCall.RunAsync(_tracer, cancellableContext, strongFingerprint, async() =>
{
using (await _lockSet.AcquireAsync(strongFingerprint.WeakFingerprint).ConfigureAwait(false))
{
var record = _records.FirstOrDefault(r => r.StrongFingerprint == strongFingerprint);
if (record == null)
{
// Match not found, add it.
record = new Record(strongFingerprint, contentHashListWithDeterminism);
_records.Add(record);
}
else
{
// Make sure we're not mixing SinglePhaseNonDeterminism records
if (record.ContentHashListWithDeterminism.Determinism.IsSinglePhaseNonDeterministic !=
contentHashListWithDeterminism.Determinism.IsSinglePhaseNonDeterministic)
{
return AddOrGetContentHashListResult.SinglePhaseMixingError;
}
// Match found.
// Replace if incoming has better determinism or some content for the existing entry is missing.
if (record.ContentHashListWithDeterminism.Determinism.ShouldBeReplacedWith(contentHashListWithDeterminism.Determinism) ||
!(await contentSession.EnsureContentIsAvailableAsync(context, record.ContentHashListWithDeterminism.ContentHashList, cts)
.ConfigureAwait(false)))
{
_records.Remove(record);
record = new Record(record.StrongFingerprint, contentHashListWithDeterminism);
_records.Add(record);
}
}
// Accept the value if it matches the final value in the cache
if (contentHashListWithDeterminism.ContentHashList.Equals(record.ContentHashListWithDeterminism.ContentHashList))
{
return new AddOrGetContentHashListResult(
new ContentHashListWithDeterminism(null, record.ContentHashListWithDeterminism.Determinism));
}
// If we didn't accept a deterministic tool's data, then we're in an inconsistent state
if (contentHashListWithDeterminism.Determinism.IsDeterministicTool)
{
return new AddOrGetContentHashListResult(
AddOrGetContentHashListResult.ResultCode.InvalidToolDeterminismError, record.ContentHashListWithDeterminism);
}
// If we did not accept the given value, return the value in the cache
return new AddOrGetContentHashListResult(record.ContentHashListWithDeterminism);
}
}));
}
}
/// <summary> /// Load a ContentHashList and the token used to replace it. /// </summary> public abstract Task <Result <(ContentHashListWithDeterminism contentHashListInfo, string replacementToken)> > GetContentHashListAsync(OperationContext context, StrongFingerprint strongFingerprint);
public async Task CacheSessionDataIsHibernated()
{
using var testDirectory = new DisposableDirectory(FileSystem);
var cacheDirectory = testDirectory.Path / "Service";
var cacheName = "theCache";
var namedCacheRoots = new Dictionary <string, AbsolutePath> {
[cacheName] = cacheDirectory / "Root"
};
var grpcPort = PortExtensions.GetNextAvailablePort();
var serverConfiguration = new LocalServerConfiguration(cacheDirectory, namedCacheRoots, grpcPort, FileSystem)
{
GrpcPortFileName = null, // Port is well known at configuration time, no need to expose it.
};
var scenario = "Default";
TaskCompletionSource <BoolResult> tcs = null;
ICache createBlockingCache(AbsolutePath path)
{
var(cache, completionSource) = CreateBlockingPublishingCache(path);
tcs = completionSource;
return(cache);
}
var server = new LocalCacheServer(
FileSystem,
TestGlobal.Logger,
scenario,
cacheFactory: createBlockingCache,
serverConfiguration,
Capabilities.All);
var context = new OperationContext(new Context(Logger));
await server.StartupAsync(context).ShouldBeSuccess();
var pat = Guid.NewGuid().ToString();
var publishingConfig = new PublishingConfigDummy
{
PublishAsynchronously = true,
};
using var clientCache = CreateClientCache(publishingConfig, pat, cacheName, grpcPort, scenario);
await clientCache.StartupAsync(context).ShouldBeSuccess();
var clientSession = clientCache.CreateSession(context, name: "TheSession", ImplicitPin.None).ShouldBeSuccess().Session;
await clientSession.StartupAsync(context).ShouldBeSuccess();
var piecesOfContent = 3;
var putResults = await Task.WhenAll(
Enumerable.Range(0, piecesOfContent)
.Select(_ => clientSession.PutRandomAsync(context, HashType.Vso0, provideHash: true, size: 128, context.Token).ShouldBeSuccess()));
var contentHashList = new ContentHashList(putResults.Select(r => r.ContentHash).ToArray());
var determinism = CacheDeterminism.ViaCache(CacheDeterminism.NewCacheGuid(), DateTime.UtcNow.AddDays(1));
var contentHashListwithDeterminism = new ContentHashListWithDeterminism(contentHashList, determinism);
var fingerprint = new Fingerprint(ContentHash.Random().ToByteArray());
var selector = new Selector(ContentHash.Random(), output: new byte[] { 0, 42 });
var strongFingerprint = new StrongFingerprint(fingerprint, selector);
var cts = new CancellationTokenSource();
// Even though publishing is blocking, this should succeed because we're publishing asynchronously.
await clientSession.AddOrGetContentHashListAsync(context, strongFingerprint, contentHashListwithDeterminism, cts.Token).ShouldBeSuccess();
// Allow for the publishing operation to be registered.
await Task.Delay(TimeSpan.FromSeconds(1));
// Simulate a restart.
await server.ShutdownAsync(context).ShouldBeSuccess();
server.Dispose();
server = new LocalCacheServer(
FileSystem,
TestGlobal.Logger,
scenario: scenario,
cacheFactory: createBlockingCache,
serverConfiguration,
Capabilities.All);
await server.StartupAsync(context).ShouldBeSuccess();
// Session should have been persisted.
var sessionsAndDatas = server.GetCurrentSessions();
sessionsAndDatas.Length.Should().Be(1);
var serverSession = sessionsAndDatas[0].session;
var data = sessionsAndDatas[0].data;
var operation = new PublishingOperation
{
ContentHashListWithDeterminism = contentHashListwithDeterminism,
StrongFingerprint = strongFingerprint
};
var operations = new[] { operation };
data.Name.Should().Be(clientSession.Name);
data.Pat.Should().Be(pat);
data.Capabilities.Should().Be(Capabilities.All);
data.ImplicitPin.Should().Be(ImplicitPin.None);
data.Pins.Should().BeEquivalentTo(new List <string>());
data.PublishingConfig.Should().BeEquivalentTo(publishingConfig);
data.PendingPublishingOperations.Should().BeEquivalentTo(operations);
var hibernateSession = serverSession as IHibernateCacheSession;
hibernateSession.Should().NotBeNull();
var actualPending = hibernateSession.GetPendingPublishingOperations();
actualPending.Should().BeEquivalentTo(operations);
// Shutting down the session should not cancel ongoing publishing operations.
await clientSession.ShutdownAsync(context).ShouldBeSuccess();
// Session should still be open in the server and operation still pending
serverSession.ShutdownStarted.Should().BeFalse();
actualPending = hibernateSession.GetPendingPublishingOperations();
actualPending.Should().BeEquivalentTo(operations);
tcs.SetResult(BoolResult.Success);
// Wait for shutdown to take place
await Task.Delay(100);
serverSession.ShutdownStarted.Should().BeTrue();
await server.ShutdownAsync(context).ShouldBeSuccess();
}
internal Task <GetContentHashListResult> GetContentHashListAsync(Context context, StrongFingerprint strongFingerprint, CancellationToken cts)
{
var ctx = new OperationContext(context, cts);
return(ctx.PerformOperationAsync(_tracer, async() =>
{
var result = await Database.GetContentHashListAsync(ctx, strongFingerprint, preferShared: false);
return result.Succeeded
? new GetContentHashListResult(result.Value.contentHashListInfo)
: new GetContentHashListResult(result);
},
extraEndMessage: _ => $"StrongFingerprint=[{strongFingerprint}]",
traceOperationStarted: false));
}
/// <summary>
/// Construct a InputAssertionList carrier object
/// </summary>
/// <param name="strongFingerprint">The strong fingerprint</param>
/// <param name="inputAssertionListContents">A string with the text of the input file list</param>
public InputAssertionList(StrongFingerprint strongFingerprint, string inputAssertionListContents)
{
StrongFingerprintValue = strongFingerprint;
InputAssertionListContents = inputAssertionListContents;
}
