private async Task ValidateSessionAsync(HashSet <FullCacheRecord> expectedRecords, ICache cache, string sessionId, FakeBuild.CasAccessMethod accessMethod)
{
if (!ImplementsTrackedSessions || DummySessionName != null)
{
return;
}
ICacheReadOnlySession readOnlySession = (await cache.CreateReadOnlySessionAsync()).Success();
// Check that the content is fine
HashSet <FullCacheRecord> foundRecords = new HashSet <FullCacheRecord>();
foreach (var strongFingerprintTask in cache.EnumerateSessionStrongFingerprints(DummySessionName ?? sessionId).Success().OutOfOrderTasks())
{
StrongFingerprint strongFingerprint = await strongFingerprintTask;
CasEntries casEntries = (await readOnlySession.GetCacheEntryAsync(strongFingerprint)).Success();
FullCacheRecord record = new FullCacheRecord(strongFingerprint, casEntries);
XAssert.IsTrue(expectedRecords.Contains(record), "Found record that was not expected!");
foundRecords.Add(record);
}
(await readOnlySession.CloseAsync()).Success();
await FakeBuild.CheckContentsAsync(cache, foundRecords, accessMethod);
XAssert.AreEqual(expectedRecords.Count, foundRecords.Count);
}
public void CasEntriesRoundTrip(int casEntryCount, int determinism)
{
CasEntries casEntries = RandomHelpers.CreateRandomCasEntries(casEntryCount, m_buildXLDeterminism[determinism]);
CasEntries roundTrip = casEntries.ToMemoization().FromMemoization();
Assert.Equal(casEntries, roundTrip);
}
public async Task <Possible <string, Failure>[]> PinToCasAsync(CasEntries casEntries, UrgencyHint urgencyHint, Guid activityId)
{
Contract.Requires(!IsClosed);
Contract.Requires(casEntries.IsValid);
using (var eventing = new PinToCasMultipleActivity(BasicFilesystemCache.EventSource, activityId, this))
{
eventing.Start(casEntries, urgencyHint);
// First, initiate all of the operations
var taskValues = new Task <Possible <string, Failure> > [casEntries.Count];
for (int i = 0; i < casEntries.Count; i++)
{
taskValues[i] = PinToCasAsync(casEntries[i], urgencyHint, activityId);
}
// Now await them all (since they can run in parallel
var results = new Possible <string, Failure> [casEntries.Count];
for (int i = 0; i < casEntries.Count; i++)
{
results[i] = await taskValues[i];
}
// All return results are actually traced via the per-hash call of PinToCas
return(eventing.Returns(results));
}
}
/// <summary>
/// Checks for discrepancies in the CasEntries between the two caches for each StrongFingerprint
/// </summary>
private IEnumerable <CacheError> CheckDeterminism()
{
var localDictionary = m_localChecker.AllFullCacheRecords;
var remoteDictionary = m_remoteChecker.AllFullCacheRecords;
foreach (var entry in localDictionary)
{
Possible <CasEntries, Failure> possibleCasEntries = entry.Value.Result;
if (!possibleCasEntries.Succeeded)
{
// A failure here indicates that the remote cache has a full cache record that the local cache does not. This is not an error.
continue;
}
CasEntries casEntries = possibleCasEntries.Result;
if (casEntries.Determinism.IsDeterministicTool)
{
if (!remoteDictionary[entry.Key].Equals(entry.Value))
{
yield return(new CacheError(CacheErrorType.DeterminismError, "Tool Determinism Error on StrongFingerprint: " + entry.Key));
}
}
else if (casEntries.Determinism.Guid.Equals(m_remoteGuid))
{
if (!remoteDictionary[entry.Key].Equals(entry.Value))
{
yield return(new CacheError(CacheErrorType.DeterminismError, "Cache Determinism Error on StrongFingerprint: " + entry.Key));
}
}
}
}
public async Task DeterminismUpgraded(int fromDeterminism, int toDeterminism, bool differentCasEntries)
{
string testName = I($"DeterminismUpgraded{fromDeterminism}x{toDeterminism}{(differentCasEntries ? "Diff" : "Same")}");
string testCacheId = MakeCacheId(testName);
ICache cache = await CreateCacheAsync(testCacheId);
string testSessionId = "Session1-" + testCacheId;
ICacheSession session = await CreateSessionAsync(cache, testSessionId);
// We need at least 2 to make "differentCasEntries" work
PipDefinition[] pips =
{
new PipDefinition("PipA", determinism: s_determinism[fromDeterminism]),
new PipDefinition("PipB", determinism: s_determinism[fromDeterminism])
};
var records = (await pips.BuildAsync(session)).ToArray();
await CloseSessionAsync(session, testSessionId);
testSessionId = "Session2-" + testCacheId;
session = await CreateSessionAsync(cache, testSessionId);
// What we will do here is AddOrGet() a record with the determinism bit changed.
for (int i = 0; i < records.Length; i++)
{
var record = records[i];
// This gets the CasEntries we want
CasEntries newEntries = records[(i + (differentCasEntries ? 1 : 0)) % records.Length].CasEntries;
// Validate that the entry for the record is what we expect
var entries = (await session.GetCacheEntryAsync(record.StrongFingerprint)).Success();
XAssert.AreEqual(s_determinism[fromDeterminism].EffectiveGuid, entries.Determinism.EffectiveGuid);
// Now pin the CasElement and all of the CasEntries
(await session.PinToCasAsync(record.StrongFingerprint.CasElement, CancellationToken.None)).Success();
(await session.PinToCasAsync(newEntries, CancellationToken.None)).Success();
// Now make a new record
var newRecord = (await session.AddOrGetAsync(
record.StrongFingerprint.WeakFingerprint,
record.StrongFingerprint.CasElement,
record.StrongFingerprint.HashElement,
new CasEntries(newEntries, s_determinism[toDeterminism]))).Success();
// The new record should be null since the determinism was upgraded
XAssert.IsNull(newRecord.Record);
// Now, we will try to get the same record from the cache to validate
// the setting of the bit
entries = (await session.GetCacheEntryAsync(record.StrongFingerprint)).Success();
XAssert.AreEqual(newEntries, entries);
XAssert.AreEqual(s_determinism[toDeterminism].EffectiveGuid, entries.Determinism.EffectiveGuid);
}
await CloseSessionAsync(session, testSessionId);
await ShutdownCacheAsync(cache, testCacheId);
}
private async Task <Possible <FullCacheRecordWithDeterminism, Failure> > AddToCache(ICacheSession session, params string[] thePaths)
{
WeakFingerprintHash weak = FakeStrongFingerprint.CreateWeak(session.CacheId);
CasHash casHash = await AddPathSet(session, thePaths);
Hash simpleHash = FakeStrongFingerprint.CreateHash(session.CacheSessionId);
return(await session.AddOrGetAsync(weak, casHash, simpleHash, CasEntries.FromCasHashes(casHash)));
}
/// <inheritdoc />
public async Task <Possible <CacheEntryPublishResult, Failure> > TryPublishCacheEntryAsync(
WeakContentFingerprint weakFingerprint,
ContentHash pathSetHash,
StrongContentFingerprint strongFingerprint,
CacheEntry entry,
CacheEntryPublishMode publishMode = CacheEntryPublishMode.CreateNew,
PublishCacheEntryOptions options = default)
{
// We can request semantics appropriate for CreateNewOrReplaceExisting via CacheDeterminism.SinglePhaseNonDeterministic
// Note that conflict-rejections / failures may still occur.
CacheDeterminism determinism = publishMode == CacheEntryPublishMode.CreateNewOrReplaceExisting
? CacheDeterminism.SinglePhaseNonDeterministic
: default(CacheDeterminism);
// N.B. this includes the metadata hash.
CasEntries adaptedHashes = new CasEntries(
entry.ToArray(h => new CasHash(new global::BuildXL.Cache.Interfaces.Hash(h))),
determinism);
Possible <FullCacheRecordWithDeterminism, Failure> maybePublished = await PerformFingerprintCacheOperationAsync(
() => m_cache.AddOrGetAsync(
weak: new WeakFingerprintHash(new Hash(weakFingerprint.Hash)),
casElement: new CasHash(new Hash(pathSetHash)),
hashElement: new Hash(strongFingerprint.Hash),
hashes: adaptedHashes,
urgencyHint: options.ShouldPublishAssociatedContent ? UrgencyHint.RegisterAssociatedContent : UrgencyHint.SkipRegisterContent),
nameof(TryPublishCacheEntryAsync));
if (maybePublished.Succeeded)
{
if (maybePublished.Result.Record == null)
{
// Happy path: Entry accepted without an alternative.
return(CacheEntryPublishResult.CreatePublishedResult());
}
else
{
// Less happy path: The underlying store has an alternative entry that we need to use instead.
Possible <CacheEntry, Failure> maybeConvertedConflictingEntry = TryConvertCasEntriesToCacheEntry(maybePublished.Result.Record.CasEntries, maybePublished.Result.Record.CacheId);
if (maybeConvertedConflictingEntry.Succeeded)
{
return(CacheEntryPublishResult.CreateConflictResult(maybeConvertedConflictingEntry.Result));
}
else
{
return(maybeConvertedConflictingEntry.Failure.Annotate(
"The cache returned a conflicting entry (rejecting the proposed entry), but the conflicting entry is invalid."));
}
}
}
else
{
return(maybePublished.Failure);
}
}
public async Task <Possible <string, Failure>[]> PinToCasAsync(CasEntries casEntries, UrgencyHint urgencyHint, Guid activityId)
{
Possible <string, Failure>[] retValues = new Possible <string, Failure> [casEntries.Count];
for (int i = 0; i < casEntries.Count; i++)
{
retValues[i] = await PinToCasAsync(casEntries[i], urgencyHint, activityId);
}
return(retValues);
}
/// <summary>
/// Check the fake build via the cache given
/// </summary>
/// <param name="cache">The cache to read from (uses a read-only session)</param>
/// <param name="index">The "index" CasHash</param>
/// <param name="entries">The CasHash entries that should match the index</param>
/// <param name="accessMethod">Method (File or stream) for how files are materialized from the cache</param>
/// <returns>Success if all worked</returns>
/// <remarks>
/// This is tied to the FakeBuild where the set of results is
/// made as the index file which we use in the strong fingerprint
/// and basically describes the set of entries that should be in the CasEntries
/// </remarks>
public static async Task CheckContentsAsync(ICache cache, CasHash index, CasEntries entries, CasAccessMethod accessMethod = CasAccessMethod.Stream)
{
Contract.Requires(cache != null);
Contract.Requires(entries.IsValid);
ICacheReadOnlySession session = await cache.CreateReadOnlySessionAsync().SuccessAsync();
await CheckContentsAsync(session, index, entries, accessMethod);
await session.CloseAsync().SuccessAsync();
}
/// <summary>
/// Checks for any errors with the cas element
/// and with each of the cas entries
/// </summary>
/// <param name="casElement">cas element of a strong fingerprint to check</param>
/// <param name="casEntries">cas entries to check</param>
/// <param name="errors">Where any cache errors found get stored</param>
private async Task CheckCas(CasHash casElement, CasEntries casEntries, ConcurrentDictionary <CacheError, int> errors)
{
// Check CasElement
await CheckCasHashAsync(casElement, errors);
for (int i = 0; i < casEntries.Count(); i++)
{
// Check each CasHash
CasHash casHash = casEntries[i];
await CheckCasHashAsync(casHash, errors);
}
}
public void CasEntriesToMemoization(int casEntryCount, int determinism)
{
CasEntries casEntries = RandomHelpers.CreateRandomCasEntries(casEntryCount, m_buildXLDeterminism[determinism]);
ContentHashListWithDeterminism contentHashListWithDeterminism = casEntries.ToMemoization();
Assert.Equal(casEntries.Count, contentHashListWithDeterminism.ContentHashList.Hashes.Count);
for (int i = 0; i < casEntries.Count; i++)
{
Assert.Equal(casEntries[i].ToMemoization(), contentHashListWithDeterminism.ContentHashList.Hashes[i]);
}
AssertDeterminismEqualEnough(casEntries.Determinism, contentHashListWithDeterminism.Determinism);
}
public Task <Possible <string, Failure>[]> PinToCasAsync(CasEntries hashes, UrgencyHint urgencyHint, Guid activityId)
{
var callback = PinToCasMultipleAsyncCallback;
if (callback != null)
{
return(callback(hashes, urgencyHint, activityId, m_realSession));
}
else
{
return(m_realSession.PinToCasAsync(hashes, urgencyHint, activityId));
}
}
public async Task NoItemFingerprint()
{
const string TestName = nameof(NoItemFingerprint);
string testCacheId = MakeCacheId(TestName);
ICache cache = await CreateCacheAsync(testCacheId);
// Now for the session (which we base on the cache ID)
string testSessionId = "Session1-" + testCacheId;
ICacheSession session = await CreateSessionAsync(cache, testSessionId);
// Note that we will be making a new fingerprint with a CasHash of NoItem
// without pre-sending it as NoItem is a special case - it is nothing
FullCacheRecord record = await FakeBuild.DoPipAsync(session, TestName);
// We place this in and did not pin the NoItem yet or even send it around
// Note that this also is doing a zero-length CasEntries
var strong = new StrongFingerprint(record.StrongFingerprint.WeakFingerprint, CasHash.NoItem, new Hash(FingerprintUtilities.ZeroFingerprint), TestName);
FullCacheRecordWithDeterminism oldRecord = (await session.AddOrGetAsync(
strong.WeakFingerprint,
strong.CasElement,
strong.HashElement,
CasEntries.FromCasHashes())).Success("Should work even though I did not pin CasHash.NoItem, instead it failed with {0}");
XAssert.IsNull(oldRecord.Record, "Should have been the first one like this");
var result = await session.GetCacheEntryAsync(strong).SuccessAsync();
XAssert.AreEqual(0, result.Count, "We should have gotten a zero-length CasEntries");
// We place this in and did not pin the NoItem yet or even send it around
// Note that this does an array of NoItem CasEntries and use the
// record.CasElement as the weak fingerprint
CasHash[] empties = { CasHash.NoItem, CasHash.NoItem, CasHash.NoItem };
strong = new StrongFingerprint(new WeakFingerprintHash(strong.CasElement.ToFingerprint()), CasHash.NoItem, new Hash(FingerprintUtilities.ZeroFingerprint), TestName);
oldRecord = (await session.AddOrGetAsync(
strong.WeakFingerprint,
CasHash.NoItem,
new Hash(FingerprintUtilities.ZeroFingerprint),
empties)).Success("Should work even though I did not pin CasHash.NoItem, instead it failed with {0}");
XAssert.IsNull(oldRecord.Record, "Should have been the first one like this");
result = await session.GetCacheEntryAsync(strong).SuccessAsync();
XAssert.AreEqual(empties, result, "We should have gotten the set of empties");
await CloseSessionAsync(session, testSessionId);
await ShutdownCacheAsync(cache, testCacheId);
}
/// <summary>
/// Writes the Start Activity event
/// </summary>
public void Start(CasEntries hashes, UrgencyHint urgencyHint)
{
Start();
if (TraceMethodArgumentsEnabled())
{
Write(
ParameterOptions,
new
{
CasEntries = hashes.ToETWFormat(),
UrgencyHint = urgencyHint,
});
}
}
public void ContentHashListWithDeterminismFromMemoization(int contentHashCount, int determinism)
{
ContentHashListWithDeterminism contentHashListWithDeterminism = new ContentHashListWithDeterminism(
ContentHashList.Random(HashingType, contentHashCount),
m_memoizationDeterminism[determinism]);
CasEntries casEntries = contentHashListWithDeterminism.FromMemoization();
Assert.Equal(contentHashListWithDeterminism.ContentHashList.Hashes.Count, casEntries.Count);
for (int i = 0; i < casEntries.Count; i++)
{
Assert.Equal(contentHashListWithDeterminism.ContentHashList.Hashes[i], casEntries[i].ToMemoization());
}
AssertDeterminismEqualEnough(casEntries.Determinism, contentHashListWithDeterminism.Determinism);
}
/// <inheritdoc />
public async Task <Possible <string, Failure>[]> PinToCasAsync(CasEntries hashes, UrgencyHint urgencyHint, Guid activityId)
{
List <ContentHash> contentHashes = hashes.Select(hash => hash.ToContentHash()).ToList();
IEnumerable <Task <Indexed <PinResult> > > resultSet = await ReadOnlyCacheSession.PinAsync(new Context(Logger), contentHashes, CancellationToken.None);
var results = new Possible <string, Failure> [contentHashes.Count];
foreach (Task <Indexed <PinResult> > resultTask in resultSet)
{
Indexed <PinResult> individualResult = await resultTask;
results[individualResult.Index] = individualResult.Item.FromMemoization(hashes[individualResult.Index], CacheId);
}
return(results);
}
public async Task <Possible <string, Failure>[]> PinToCasAsync(CasEntries hashes, UrgencyHint urgencyHint, Guid activityId)
{
using (var eventing = new PinToCasMultipleActivity(CompositingCache.EventSource, activityId, this))
{
eventing.Start(hashes, urgencyHint);
var results = await m_casSession.PinToCasAsync(hashes, urgencyHint, eventing.Id);
for (int i = 0; i < results.Length; i++)
{
if (results[i].Succeeded)
{
PinnedToCas.TryAdd(hashes[i], 0);
}
}
return(eventing.Returns(results));
}
}
private static async Task <FullCacheRecord> DoPipAsyncImpl(ICacheSession session,
WeakFingerprintHash weak,
Hash simpleHash,
FakeBuild fake,
CacheDeterminism determinism,
CasAccessMethod accessMethod)
{
foreach (var strongTask in session.EnumerateStrongFingerprints(weak))
{
StrongFingerprint possibleHit = await strongTask.SuccessAsync();
if (fake.OutputListHash.Equals(possibleHit.CasElement))
{
if (simpleHash.Equals(possibleHit.HashElement))
{
// A cache hit! Our strong fingerprint matched
return(new FullCacheRecord(possibleHit, await session.GetCacheEntryAsync(possibleHit).SuccessAsync()));
}
}
}
// A cache miss - add the content to the cache and then
// add the build.
CasHash inputList = await AddToCasAsync(fake.OutputList, accessMethod, session).SuccessAsync();
CasHash[] items = new CasHash[fake.Outputs.Length];
for (int i = 0; i < items.Length; i++)
{
items[i] = await AddToCasAsync(fake.Outputs[i], accessMethod, session).SuccessAsync();
}
CasEntries entries = new CasEntries(items, determinism);
FullCacheRecordWithDeterminism cacheRecord = await session.AddOrGetAsync(weak, inputList, simpleHash, entries).SuccessAsync();
XAssert.AreEqual(null, cacheRecord.Record);
// Produce a full cache record manually - such that the CacheId is our own "NewRecordCacheId"
return(new FullCacheRecord(new StrongFingerprint(weak, inputList, simpleHash, NewRecordCacheId), entries));
}
/// <summary>
/// Writes the Start Activity event
/// </summary>
public void Start(WeakFingerprintHash weak, CasHash casElement, Hash hashElement, CasEntries hashes, UrgencyHint urgencyHint)
{
Start();
if (TraceMethodArgumentsEnabled())
{
Write(
ParameterOptions,
new
{
WeakFingerprintHash = weak,
CasHash = casElement,
Hash = hashElement,
CasEntries = hashes,
UrgencyHint = urgencyHint,
});
}
}
public async Task SimpleDummySession()
{
const string TestName = "SimpleSession";
string testCacheId = MakeCacheId(TestName);
ICache cache = await CreateCacheAsync(testCacheId);
// Now for the session (which we base on the cache ID)
string testSessionId = "Session1-" + testCacheId;
ICacheSession session = await CreateSessionAsync(cache, testSessionId);
// Do the default fake build for this test (first time, no cache hit)
FullCacheRecord built = await FakeBuild.DoPipAsync(session, TestName);
XAssert.AreEqual(FakeBuild.NewRecordCacheId, built.CacheId, "Should have been a new cache entry!");
// Now we see if we can get back the items we think we should
await CloseSessionAsync(session, testSessionId);
// We need a read only session to get the CasEntries
ICacheReadOnlySession readOnlySession = (await cache.CreateReadOnlySessionAsync()).Success();
// Validate that the cache contains a dummy session and it has the one cache record it needs.
HashSet <FullCacheRecord> found = new HashSet <FullCacheRecord>();
foreach (var strongFingerprintTask in cache.EnumerateSessionStrongFingerprints(MemoizationStoreAdapterCache.DummySessionName).Success().OutOfOrderTasks())
{
StrongFingerprint strongFingerprint = await strongFingerprintTask;
CasEntries casEntries = (await readOnlySession.GetCacheEntryAsync(strongFingerprint)).Success();
FullCacheRecord record = new FullCacheRecord(strongFingerprint, casEntries);
// If it is not the record we already found...
if (!found.Contains(record))
{
found.Add(record);
}
XAssert.AreEqual(1, found.Count, "There should be only 1 unique record in the session");
XAssert.AreEqual(built.StrongFingerprint.WeakFingerprint, record.StrongFingerprint.WeakFingerprint);
XAssert.AreEqual(built.StrongFingerprint.CasElement, record.StrongFingerprint.CasElement);
XAssert.AreEqual(built.StrongFingerprint.HashElement, record.StrongFingerprint.HashElement);
XAssert.AreEqual(built.CasEntries.Count, record.CasEntries.Count, "Did not return the same number of items");
XAssert.IsTrue(record.CasEntries.Equals(built.CasEntries), "Items returned are not the same hash and/or order order");
XAssert.AreEqual(built, record);
// We can not check record.CasEntries.IsDeterministic
// as the cache may have determined that they are deterministic
// via cache determinism recovery.
}
XAssert.AreEqual(1, found.Count, "There should be 1 and only 1 record in the session!");
await readOnlySession.CloseAsync().SuccessAsync();
// Check that the cache has the items in it
await FakeBuild.CheckContentsAsync(cache, built);
// Now redo the "build" with a cache hit
testSessionId = "Session2-" + testCacheId;
session = await CreateSessionAsync(cache, testSessionId);
FullCacheRecord rebuilt = await FakeBuild.DoPipAsync(session, TestName);
XAssert.AreEqual(built, rebuilt, "Should have been the same build!");
// We make sure we did get it from a cache rather than a manual rebuild.
XAssert.AreNotEqual(built.CacheId, rebuilt.CacheId, "Should not be the same cache ID");
await CloseSessionAsync(session, testSessionId);
readOnlySession = await cache.CreateReadOnlySessionAsync().SuccessAsync();
// Now that we have done the second build via a cache hit, it should produce the
// same cache record as before
foreach (var strongFingerprintTask in cache.EnumerateSessionStrongFingerprints(MemoizationStoreAdapterCache.DummySessionName).Success().OutOfOrderTasks())
{
StrongFingerprint strongFingerprint = await strongFingerprintTask;
CasEntries casEntries = (await readOnlySession.GetCacheEntryAsync(strongFingerprint)).Success();
FullCacheRecord record = new FullCacheRecord(strongFingerprint, casEntries);
XAssert.IsTrue(found.Contains(record), "Second session should produce the same cache record but did not!");
}
(await readOnlySession.CloseAsync()).Success();
await ShutdownCacheAsync(cache, testCacheId);
}
public Task <Possible <FullCacheRecordWithDeterminism, Failure> > AddOrGetAsync(WeakFingerprintHash weak, CasHash casElement, Hash hashElement, CasEntries hashes, UrgencyHint urgencyHint, Guid activityId)
{
var callback = AddOrGetAsyncCallback;
if (callback != null)
{
return(callback(weak, casElement, hashElement, hashes, urgencyHint, activityId, m_realSession));
}
else
{
return(m_realSession.AddOrGetAsync(weak, casElement, hashElement, hashes, urgencyHint, activityId));
}
}
public Task <Possible <string, Failure>[]> PinToCasAsync(CasEntries hashes, UrgencyHint urgencyHint, Guid activityId)
{
return(m_session.PinToCasAsync(hashes, urgencyHint, activityId));
}
public async Task <Possible <FullCacheRecordWithDeterminism, Failure> > AddOrGetAsync(WeakFingerprintHash weak, CasHash casElement, Hash hashElement, CasEntries hashes, UrgencyHint urgencyHint, Guid activityId)
{
var check = await CheckInputList(weak, casElement, hashElement, urgencyHint, activityId);
if (!check.Succeeded)
{
return(check.Failure);
}
return(await m_session.AddOrGetAsync(weak, casElement, hashElement, hashes, urgencyHint, activityId));
}
public async Task <Possible <FullCacheRecordWithDeterminism, Failure> > AddOrGetAsync(WeakFingerprintHash weak, CasHash casElement, Hash hashElement, CasEntries hashes, UrgencyHint urgencyHint, Guid activityId)
{
var addResult = await CacheSession.AddOrGetContentHashListAsync(
new Context(Logger),
new BuildXL.Cache.MemoizationStore.Interfaces.Sessions.StrongFingerprint(
weak.ToMemoization(),
new Selector(casElement.ToMemoization(), hashElement.RawHash.ToByteArray())),
hashes.ToMemoization(),
CancellationToken.None);
var strong = new StrongFingerprint(weak, casElement, hashElement, CacheId);
switch (addResult.Code)
{
case AddOrGetContentHashListResult.ResultCode.Success:
SessionEntries?.TryAdd(strong, 1);
return(addResult.ContentHashListWithDeterminism.ContentHashList == null
? new FullCacheRecordWithDeterminism(addResult.ContentHashListWithDeterminism.Determinism.FromMemoization())
: new FullCacheRecordWithDeterminism(new FullCacheRecord(strong, addResult.ContentHashListWithDeterminism.FromMemoization())));
case AddOrGetContentHashListResult.ResultCode.SinglePhaseMixingError:
return(new SinglePhaseMixingFailure(CacheId));
case AddOrGetContentHashListResult.ResultCode.InvalidToolDeterminismError:
return(new NotDeterministicFailure(
CacheId,
new FullCacheRecord(strong, addResult.ContentHashListWithDeterminism.FromMemoization()),
new FullCacheRecord(strong, hashes)));
case AddOrGetContentHashListResult.ResultCode.Error:
return(new CacheFailure(addResult.ErrorMessage));
default:
return(new CacheFailure("Unrecognized AddOrGetContentHashListAsync result code: " + addResult.Code + ", error message: " + (addResult.ErrorMessage ?? string.Empty)));
}
}
public async Task <Possible <FullCacheRecordWithDeterminism, Failure> > AddOrGetAsync(WeakFingerprintHash weak, CasHash casElement, Hash hashElement, CasEntries hashes, UrgencyHint urgencyHint, Guid activityId)
{
using (var eventing = new AddOrGetActivity(CompositingCache.EventSource, activityId, this))
{
eventing.Start(weak, casElement, hashElement, hashes, urgencyHint);
// First, check if all the content has been pinned.
if (Cache.StrictMetadataCasCoupling)
{
List <CasHash> referencedHashes = new List <CasHash>(hashes);
referencedHashes.Add(casElement);
foreach (CasHash oneHash in referencedHashes)
{
if (!PinnedToCas.ContainsKey(oneHash))
{
return(eventing.StopFailure(new UnpinnedCasEntryFailure(Cache.CacheId, oneHash)));
}
}
}
// Then check the determinism bit...
var testStrongFingerprint = new StrongFingerprint(weak, casElement, hashElement, Cache.CacheId);
var existRecordCheck = await m_metadataSession.GetCacheEntryAsync(testStrongFingerprint, urgencyHint, eventing.Id);
if (existRecordCheck.Succeeded)
{
// There's an existing record, so we need the rules for determinism upgrade / downgrade. Which are complicated by the existance
// (or lack there of) of the associated content.
// First we'll check for determinism upgrades or other situations that don't require content probes.
var existingRecord = existRecordCheck.Result;
// If the determinsim is going from no determinism to some determinism OR the record is going from a non-tool
// determinism to a tool determinism, replace
if ((existingRecord.Determinism.IsNone && hashes.Determinism.IsDeterministic) ||
(hashes.Determinism.IsDeterministicTool && !existingRecord.Determinism.IsDeterministicTool))
{
return(eventing.Returns(await m_metadataSession.AddOrGetAsync(weak, casElement, hashElement, hashes, urgencyHint, eventing.Id)));
}
// Before trying to probe for files, see if the entries are identical.
// If so, tell the caller we stored their record.
if (hashes == existingRecord && hashes.Determinism.EffectiveGuid == existingRecord.Determinism.EffectiveGuid)
{
return(eventing.Returns(new FullCacheRecordWithDeterminism(CacheDeterminism.None)));
}
// The rest of the determinism moves depend on if the files exist or not.
bool foundAllFiles = true;
var pinCheck = await m_casSession.PinToCasAsync(existRecordCheck.Result, urgencyHint, eventing.Id);
foreach (var oneOutput in pinCheck)
{
if (!oneOutput.Succeeded)
{
foundAllFiles = false;
}
}
// Ok, so now with the knowledge of file existance, if we're missing files, allow the call through no matter what.
if (!foundAllFiles)
{
return(eventing.Returns(await m_metadataSession.AddOrGetAsync(weak, casElement, hashElement, hashes, urgencyHint, eventing.Id)));
}
// If the existing entry is more deterministic than what we're being given or
// the records are the same level, unless tool deterministic or single phase.
if ((existingRecord.Determinism.IsDeterministic && !hashes.Determinism.IsDeterministic) ||
(existingRecord.Determinism.IsDeterministicTool && !hashes.Determinism.IsDeterministicTool) ||
(existingRecord.Determinism.EffectiveGuid == hashes.Determinism.EffectiveGuid && !existingRecord.Determinism.IsDeterministicTool && !existingRecord.Determinism.IsSinglePhaseNonDeterministic))
{
// If the records are identical except for determinsim, return null.
if (existingRecord == hashes)
{
return(eventing.Returns(new FullCacheRecordWithDeterminism(CacheDeterminism.None)));
}
return(eventing.Returns(new FullCacheRecordWithDeterminism(new FullCacheRecord(testStrongFingerprint, existingRecord))));
}
// And now the error conditions.
// If a tool determinism collection, or an attempt to go from deterministic to not deterministic.
if (existingRecord.Determinism.IsDeterministicTool && hashes.Determinism.IsDeterministicTool)
{
return(eventing.StopFailure(new NotDeterministicFailure(Cache.CacheId, new FullCacheRecord(testStrongFingerprint, existingRecord), new FullCacheRecord(testStrongFingerprint, hashes))));
}
}
return(eventing.Returns(await m_metadataSession.AddOrGetAsync(weak, casElement, hashElement, hashes, urgencyHint, eventing.Id)));
}
}
private Possible <FullCacheRecordWithDeterminism, Failure> AddOrGet(WeakFingerprintHash weak, CasHash casElement, BuildXL.Cache.Interfaces.Hash hashElement, CasEntries hashes)
{
Contract.Requires(!IsClosed);
Contract.Requires(hashes.IsValid);
Contract.Assert(!m_readOnly);
// We check the Cas entries if we are strict
if (StrictMetadataCasCoupling)
{
// Check that the content is valid.
if (!m_pinnedToCas.ContainsKey(casElement))
{
return(new UnpinnedCasEntryFailure(CacheId, casElement));
}
foreach (CasHash hash in hashes)
{
if (!m_pinnedToCas.ContainsKey(hash))
{
return(new UnpinnedCasEntryFailure(CacheId, hash));
}
}
}
var strongFingerprints = Cache.Fingerprints.GetOrAdd(weak, (key) => new ConcurrentDictionary <StrongFingerprint, FullCacheRecord>());
var record = strongFingerprints.AddOrUpdate(
new StrongFingerprint(weak, casElement, hashElement, Cache.CacheId),
(strong) => new FullCacheRecord(strong, hashes),
(strong, oldRecord) =>
{
// Do no harm here - we will recheck this outside to produce the error that it was a bad attempt
if (oldRecord.CasEntries.Determinism.IsSinglePhaseNonDeterministic != hashes.Determinism.IsSinglePhaseNonDeterministic)
{
return(oldRecord);
}
// We replace if we are SinglePhaseNonDeterministic *or*
// if we are upgrading the determinism.
if (hashes.Determinism.IsSinglePhaseNonDeterministic ||
(!oldRecord.CasEntries.Determinism.IsDeterministicTool &&
(hashes.Determinism.IsDeterministic && !hashes.Determinism.Equals(oldRecord.CasEntries.Determinism))) ||
HasMissingContent(oldRecord.CasEntries))
{
oldRecord = new FullCacheRecord(strong, hashes);
}
return(oldRecord);
});
if (record.CasEntries.Determinism.IsSinglePhaseNonDeterministic != hashes.Determinism.IsSinglePhaseNonDeterministic)
{
return(new SinglePhaseMixingFailure(CacheId));
}
AddSessionRecord(record);
if (record.CasEntries.Equals(hashes))
{
record = null;
}
else
{
// Check if tool determinism did not make it in - that means a very bad thing happened
if (hashes.Determinism.IsDeterministicTool)
{
return(new NotDeterministicFailure(Cache.CacheId, record, new FullCacheRecord(record.StrongFingerprint, hashes)));
}
}
if (record == null)
{
return(new FullCacheRecordWithDeterminism(hashes.GetFinalDeterminism(Cache.IsAuthoritative, Cache.CacheGuid, CacheDeterminism.NeverExpires)));
}
else
{
return(new FullCacheRecordWithDeterminism(new FullCacheRecord(record.StrongFingerprint, record.CasEntries.GetModifiedCasEntriesWithDeterminism(Cache.IsAuthoritative, Cache.CacheGuid, CacheDeterminism.NeverExpires))));
}
}
public Task <Possible <FullCacheRecordWithDeterminism, Failure> > AddOrGetAsync(WeakFingerprintHash weak, CasHash casElement, BuildXL.Cache.Interfaces.Hash hashElement, CasEntries hashes, UrgencyHint urgencyHint, Guid activityId)
{
Contract.Requires(!IsClosed);
Contract.Requires(hashes.IsValid);
Contract.Assert(!m_readOnly);
return(Task.Run(() => AddOrGet(weak, casElement, hashElement, hashes)));
}
public async Task CasEntriesReplacedOnMissingContent(int fromDeterminism, int toDeterminism, bool differentCasEntries)
{
string testName = I($"ReplacedOnMissingContent{fromDeterminism}x{toDeterminism}{(differentCasEntries ? "Diff" : "Same")}");
string testCacheId = MakeCacheId(testName);
ICache cache = await CreateCacheAsync(testCacheId, strictMetadataCasCoupling : false);
try
{
string testSessionId = "Session1-" + testCacheId;
ICacheSession session = await CreateSessionAsync(cache, testSessionId);
// We need at least 2 to make "differentCasEntries" work
FullCacheRecord[] records =
{
RandomHelpers.CreateRandomFullCacheRecord(session.CacheId, s_determinism[fromDeterminism]),
RandomHelpers.CreateRandomFullCacheRecord(session.CacheId, s_determinism[fromDeterminism])
};
foreach (var record in records)
{
var addResult = await session.AddOrGetAsync(
record.StrongFingerprint.WeakFingerprint,
record.StrongFingerprint.CasElement,
record.StrongFingerprint.HashElement,
record.CasEntries).SuccessAsync();
XAssert.IsNull(addResult.Record);
}
await CloseSessionAsync(session, testSessionId);
testSessionId = "Session2-" + testCacheId;
session = await CreateSessionAsync(cache, testSessionId);
// What we will do here is AddOrGet() a record with the determinism bit changed.
for (int i = 0; i < records.Length; i++)
{
var record = records[i];
var getResult = await session.GetCacheEntryAsync(record.StrongFingerprint).SuccessAsync();
XAssert.AreEqual(record.CasEntries.Determinism.EffectiveGuid, getResult.Determinism.EffectiveGuid);
// This gets the CasEntries we want
var recordsLength = records.Length;
CasEntries newEntries = records[(i + (differentCasEntries ? 1 : 0)) % recordsLength].CasEntries;
// Validate that the entry for the record is what we expect
var entries = (await session.GetCacheEntryAsync(record.StrongFingerprint)).Success();
XAssert.AreEqual(s_determinism[fromDeterminism].EffectiveGuid, entries.Determinism.EffectiveGuid);
// Now make a new record
var newRecord = (await session.AddOrGetAsync(
record.StrongFingerprint.WeakFingerprint,
record.StrongFingerprint.CasElement,
record.StrongFingerprint.HashElement,
new CasEntries(newEntries, s_determinism[toDeterminism]))).Success();
// The new record should be null because the old value should have
// been replaced with the new value in all cases (due to missing content).
XAssert.IsNull(newRecord.Record);
// Now, we will try to get the same record from the cache to validate the replacement
entries = (await session.GetCacheEntryAsync(record.StrongFingerprint)).Success();
XAssert.AreEqual(newEntries, entries);
XAssert.AreEqual(s_determinism[toDeterminism].EffectiveGuid, entries.Determinism.EffectiveGuid);
}
await CloseSessionAsync(session, testSessionId);
}
finally
{
await ShutdownCacheAsync(cache, testCacheId);
}
}
public async Task <Possible <FullCacheRecordWithDeterminism, Failure> > AddOrGetAsync(WeakFingerprintHash weak, CasHash casElement, Hash hashElement, CasEntries hashes, UrgencyHint urgencyHint, Guid activityId)
{
Contract.Requires(!IsClosed);
Contract.Requires(hashes.IsValid);
Contract.Assert(!IsReadOnly);
using (var counter = m_counters.AddOrGetCounter())
{
using (var eventing = new AddOrGetActivity(BasicFilesystemCache.EventSource, activityId, this))
{
eventing.Start(weak, casElement, hashElement, hashes, urgencyHint);
counter.SetEntriesCount(hashes.Count); // The size of what we are adding (effectively)
// We check the Cas entries if we are strict
if (StrictMetadataCasCoupling)
{
// Check that the content is valid.
if (!m_pinnedToCas.ContainsKey(casElement))
{
counter.Failed();
return(eventing.StopFailure(new UnpinnedCasEntryFailure(CacheId, casElement)));
}
foreach (CasHash hash in hashes)
{
if (!m_pinnedToCas.ContainsKey(hash))
{
counter.Failed();
return(eventing.StopFailure(new UnpinnedCasEntryFailure(CacheId, hash)));
}
}
}
StrongFingerprint strong = new StrongFingerprint(weak, casElement, hashElement, CacheId);
// Assume we accepted the Add and there is nothing to return
FullCacheRecord result = null;
string strongFingerprintName = m_cache.GetStrongFingerprintFilename(strong);
try
{
using (FileStream file = await m_cache.ContendedOpenStreamAsync(strongFingerprintName, FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
// The compiler thinks that it is not assigned down below at the end
// even though it would be set by the writeEntries being true in that case
CasEntries oldCasEntries = hashes;
// Assume we will write our new enties to the file.
bool writeEntries = true;
// If there is some data in the file already, we need to try to read it.
if (file.Length > 0)
{
var possibleOldCasEntries = await m_cache.ReadCacheEntryAsync(file);
// Only if it was formatted correctly do we continue to check if
// we should replace it.
if (possibleOldCasEntries.Succeeded)
{
oldCasEntries = possibleOldCasEntries.Result;
writeEntries = false;
// We can only replace if both or neither is SinglePhaseNonDeterministic
if (oldCasEntries.Determinism.IsSinglePhaseNonDeterministic != hashes.Determinism.IsSinglePhaseNonDeterministic)
{
counter.Failed();
return(eventing.StopFailure(new SinglePhaseMixingFailure(CacheId)));
}
// Should we replace?
if (hashes.Determinism.IsSinglePhaseNonDeterministic ||
(!oldCasEntries.Determinism.IsDeterministicTool &&
(hashes.Determinism.IsDeterministic && !oldCasEntries.Determinism.Equals(hashes.Determinism))))
{
// We are replacing due to determinism
counter.Det();
writeEntries = true;
}
else if (HasMissingContent(oldCasEntries))
{
counter.Repair();
writeEntries = true;
}
else if (oldCasEntries.Determinism.IsDeterministicTool && hashes.Determinism.IsDeterministicTool && !oldCasEntries.Equals(hashes))
{
// We have a non-deterministic tool!
counter.Failed();
return(eventing.StopFailure(new NotDeterministicFailure(CacheId, new FullCacheRecord(strong, oldCasEntries), new FullCacheRecord(strong, hashes))));
}
}
}
// Are we going to write the entries?
if (writeEntries)
{
// We are writing so the old entries don't count
oldCasEntries = hashes;
// Write from the front
file.SetLength(0);
await m_cache.WriteCacheEntryAsync(file, hashes);
}
else
{
counter.Dup();
}
// If what is in the cache is different than what we are
// asking to add, build a FullCacheRecord to return what
// is in the cache.
if (!oldCasEntries.Equals(hashes))
{
counter.Get();
result = new FullCacheRecord(strong, oldCasEntries);
}
}
}
catch (Exception e)
{
counter.Failed();
return(eventing.StopFailure(new StrongFingerprintAccessFailure(m_cache.CacheId, strong, e)));
}
AddSessionRecord(strong);
if (result != null)
{
return
(eventing.Returns(
new FullCacheRecordWithDeterminism(
new FullCacheRecord(
result.StrongFingerprint,
result.CasEntries.GetModifiedCasEntriesWithDeterminism(
m_cache.IsAuthoritative,
m_cache.CacheGuid,
CacheDeterminism.NeverExpires)))));
}
else
{
return(eventing.Returns(new FullCacheRecordWithDeterminism(hashes.GetFinalDeterminism(m_cache.IsAuthoritative, m_cache.CacheGuid, DateTime.UtcNow.Add(m_cache.TimeToLive)))));
}
}
}
}
/// <summary>
/// Check the fake build via the session given
/// </summary>
/// <param name="session">Session to use for the check</param>
/// <param name="index">The "index" CasHash</param>
/// <param name="entries">The CasHash entries that should match the index</param>
/// <param name="accessMethod">Method (File or stream) for how files are materialized from the cache</param>
/// <returns>An Async Task</returns>
public static async Task CheckContentsAsync(ICacheReadOnlySession session, CasHash index, CasEntries entries, CasAccessMethod accessMethod = CasAccessMethod.Stream)
{
string cacheId = await session.PinToCasAsync(index, CancellationToken.None).SuccessAsync("Cannot pin entry {0} to cache {1}", index.ToString(), session.CacheId);
string[] expected = (await GetStreamAsync(index, accessMethod, session)).Success().Stream.AsString().Split(s_splitLines, StringSplitOptions.RemoveEmptyEntries);
XAssert.AreEqual(expected.Length, entries.Count, "Counts did not match from cache {0}: {1} != {2}", cacheId, expected.Length, entries.Count);
for (int i = 0; i < expected.Length; i++)
{
string casCacheId = await session.PinToCasAsync(entries[i], CancellationToken.None).SuccessAsync();
string entry = (await GetStreamAsync(entries[i], accessMethod, session)).Success().Stream.AsString();
XAssert.AreEqual(expected[i], entry, "CasEntry {0} mismatch from cache {1}: [{2}] != [{3}]", i, casCacheId, expected[i], entry);
}
}
