/// <summary>
/// Attempts to retrieve a fingerprint entry.
/// If the query is successful, a <see cref="PipFingerprintEntry"/> is returned (or null, if there is no current entry for the fingerprint).
/// </summary>
public async Task <Possible <PipFingerprintEntry, Failure> > TryGetFingerprintEntryAsync(ContentFingerprint fingerprint, CacheQueryData cacheQueryData = null)
{
StrongContentFingerprint dummyFingerprint = ComputeDummyStrongFingerprint(
m_pathTable,
fingerprint);
var weakFingerprint = new WeakContentFingerprint(fingerprint.Hash);
Possible <CacheEntry?, Failure> maybeEntry = await m_twoPhaseStore.TryGetCacheEntryAsync(
weakFingerprint : weakFingerprint,
pathSetHash : s_dummyPathSetHash,
strongFingerprint : dummyFingerprint);
if (!maybeEntry.Succeeded)
{
return(maybeEntry.Failure);
}
if (!maybeEntry.Result.HasValue)
{
// Real miss.
return((PipFingerprintEntry)null);
}
if (cacheQueryData != null)
{
cacheQueryData.WeakContentFingerprint = weakFingerprint;
cacheQueryData.PathSetHash = s_dummyPathSetHash;
cacheQueryData.StrongContentFingerprint = dummyFingerprint;
cacheQueryData.MetadataHash = maybeEntry.Result.Value.MetadataHash;
cacheQueryData.ContentCache = m_contentCache;
}
return(await TryLoadAndDeserializeContent(maybeEntry.Result.Value.MetadataHash));
}
/// <summary>
/// See <see cref="ITwoPhaseFingerprintStore.TryPublishCacheEntryAsync"/>
/// </summary>
public virtual async Task <Possible <CacheEntryPublishResult, Failure> > TryPublishCacheEntryAsync(
Pip pip,
WeakContentFingerprint weakFingerprint,
ContentHash pathSetHash,
StrongContentFingerprint strongFingerprint,
CacheEntry entry,
CacheEntryPublishMode mode = CacheEntryPublishMode.CreateNew)
{
Contract.Requires(pathSetHash.IsValid);
Contract.Requires(entry.MetadataHash.IsValid);
var result = await TwoPhaseFingerprintStore.TryPublishCacheEntryAsync(
weakFingerprint,
pathSetHash,
strongFingerprint,
entry,
mode);
if (result.Succeeded)
{
var publishedEntry = result.Result.Status == CacheEntryPublishStatus.Published ? entry : result.Result.ConflictingEntry;
Tracing.Logger.Log.PipTwoPhaseCachePublishCacheEntry(
LoggingContext,
pip.GetDescription(Context),
weakFingerprint.ToString(),
pathSetHash.ToString(),
strongFingerprint.ToString(),
entry.MetadataHash.ToString(),
result.Result.Status.ToString(),
publishedEntry.MetadataHash.ToString());
}
return(result);
}
public async Task RoundtripCacheEntryWithMetadata()
{
ContentHash pathSetHash = await AddContent("This is a list of paths");
WeakContentFingerprint weak = CreateWeakFingerprint("Fingerprint text here");
StrongContentFingerprint strong = CreateStrongFingerprint("Strong fingerprint text here");
CacheEntry entry = await CreateCacheEntryAndStoreContent("Metadata", "A", "B");
Possible <CacheEntryPublishResult> maybePublished = await FingerprintStore.TryPublishCacheEntryAsync(
weak,
pathSetHash,
strong,
entry);
XAssert.IsTrue(maybePublished.Succeeded);
CacheEntryPublishResult publishResult = maybePublished.Result;
XAssert.AreEqual(CacheEntryPublishStatus.Published, publishResult.Status);
Possible <CacheEntry?> maybeFetched = await FingerprintStore.TryGetCacheEntryAsync(
weak,
pathSetHash,
strong);
XAssert.IsTrue(maybeFetched.Succeeded);
XAssert.IsTrue(maybeFetched.Result.HasValue, "Unexpected miss (was just stored)");
CacheEntry roundtrippedEntry = maybeFetched.Result.Value;
AssertCacheEntriesEquivalent(entry, roundtrippedEntry);
}
/// <inheritdoc />
public void DeserializeAndUpdate(BinaryLogReader.EventReader reader)
{
PipId = new PipId(reader.ReadUInt32Compact());
Kind = (FingerprintComputationKind)reader.ReadInt32Compact();
WeakFingerprint = reader.ReadWeakFingerprint();
StrongFingerprintComputations = reader.ReadReadOnlyList(r => new ProcessStrongFingerprintComputationData((BinaryLogReader.EventReader)r));
}
/// <summary>
/// Helper functions for putting entries into the fingerprint store for sub-components.
/// { pip formatted semi stable hash : weak fingerprint, strong fingerprint, path set hash }
/// { weak fingerprint hash : weak fingerprint inputs }
/// { strong fingerprint hash : strong fingerprint inputs }
/// { path set hash : path set inputs }
/// </summary>
private FingerprintStoreEntry CreateAndStoreFingerprintStoreEntry(
FingerprintStore fingerprintStore,
Process pip,
PipFingerprintKeys pipFingerprintKeys,
WeakContentFingerprint weakFingerprint,
ProcessStrongFingerprintComputationData strongFingerprintData,
bool cacheWeakFingerprintSerialization = false)
{
// If we got this far, a new pip is being put in the store
Counters.IncrementCounter(FingerprintStoreCounters.NumPipFingerprintEntriesPut);
UpdateOrStorePipUniqueOutputHashEntry(fingerprintStore, pip);
// A content hash-keyed entry will have the same value as long as the key is the same, so overwriting it is unnecessary
var mustStorePathEntry = !fingerprintStore.ContainsContentHash(pipFingerprintKeys.FormattedPathSetHash) || CacheMissAnalysisEnabled;
var entry = CreateFingerprintStoreEntry(
pip,
pipFingerprintKeys,
weakFingerprint,
strongFingerprintData,
mustStorePathEntry: mustStorePathEntry,
cacheWeakFingerprintSerialization: cacheWeakFingerprintSerialization);
fingerprintStore.PutFingerprintStoreEntry(entry, storePathSet: mustStorePathEntry);
return(entry);
}
/// <summary>
/// Gets the deserialized path set given its content hash
/// </summary>
public virtual async Task <Possible <ObservedPathSet> > TryRetrievePathSetAsync(
OperationContext operationContext,
// TODO: Do we need this fingerprint given that the path set hash is provided by this interface in the first place
WeakContentFingerprint weakFingerprint,
ContentHash pathSetHash)
{
using (operationContext.StartOperation(PipExecutorCounter.TryLoadPathSetFromContentCacheDuration))
{
Possible <Stream> maybePathSetStream = await TryLoadAndOpenPathSetStreamAsync(pathSetHash);
if (!maybePathSetStream.Succeeded)
{
return(maybePathSetStream.Failure);
}
using (operationContext.StartOperation(PipExecutorCounter.TryLoadPathSetFromContentCacheDeserializeDuration))
using (var pathSetReader = new BuildXLReader(debug: false, stream: maybePathSetStream.Result, leaveOpen: false))
{
var maybeDeserialized = ObservedPathSet.TryDeserialize(PathTable, pathSetReader, m_pathExpander);
if (!maybeDeserialized.Succeeded)
{
return(maybeDeserialized.Failure);
}
return(maybeDeserialized.Result);
}
}
}
public void AssertFingerprintIsNew(WeakContentFingerprint weakFingerprint, params ObservedInput[] inputs)
{
var result = CreateResult(PathTable, inputs);
StrongContentFingerprint fp = result.ComputeStrongFingerprint(PathTable, weakFingerprint, ContentHashingUtilities.ZeroHash);
XAssert.IsFalse(Fingerprints.Contains(fp), "Duplicate strong fingerprint");
Fingerprints.Add(fp);
}
/// <inheritdoc />
public Task <Possible <CacheEntryPublishResult, Failure> > TryPublishCacheEntryAsync(
WeakContentFingerprint weakFingerprint,
ContentHash pathSetHash,
StrongContentFingerprint strongFingerprint,
CacheEntry entry,
CacheEntryPublishMode mode = CacheEntryPublishMode.CreateNew,
PublishCacheEntryOptions options = default)
{
return(Task.FromResult(new Possible <CacheEntryPublishResult, Failure>(CacheEntryPublishResult.CreatePublishedResult())));
}
private static (WeakContentFingerprint wf, StrongContentFingerprint sf) GetBuildManifestHashKey(ContentHash hash)
{
var hashBytes = hash.ToByteArray();
Array.Resize(ref hashBytes, FingerprintUtilities.FingerprintLength);
var wf = new WeakContentFingerprint(FingerprintUtilities.CreateFrom(hashBytes));
var sf = new StrongContentFingerprint(wf.Hash);
return(wf, sf);
}
/// <inheritdoc />
public async Task <Possible <CacheEntry?, Failure> > TryGetCacheEntryAsync(
WeakContentFingerprint weakFingerprint,
ContentHash pathSetHash,
StrongContentFingerprint strongFingerprint)
{
// TODO: We need a different side channel for prefetching etc. other than strong fingerprint subclasses.
// - Given aggregation of multiple *closely aligned* stores as the common case,
// one has the problem of optimizing which instance you hand to which store (and if you get it wrong, maybe things are just slower).
// - For it to work, we have to be rather disingenuous about equality (see examples below).
// - Usage this way is extremely error prone (with a perf pit if you get it wrong:
//
// foreach (StrongFingerprint candidate in Enumerate(...)) {
// StrongFingerprint satisifiable = ComputeAvailable(LoadPathSet(candidate.CasElement));
// if (satisfiable == candidate) { // TODO: We have to lie about equality!
// GetCacheEntryAndRunPipFromCache(satisfiable); // TODO: Oops. Accidentally slower?
// }
// }
//
// - By itself, lying about equality (for sake of the side-channel) probably has unintended consequences:
// FancyStrongFingerprintCache.TryGetValue(pathSetHash, out strongFingerprint); // TODO: Oops. Side channel is broken again.
//
// - The implementations have to figure out which StrongFingerprints have side channel info for them. This requires
// reflecting at least; but implementations must be vigilant to StrongFingerprints from *other instances*. This can break invariants,
// (e.g. what if someone had a dictionary of 'fingerprints I totally returned' to some other data, and indexed blindly), and means we
// have more to worry about w.r.t. cohabitation of implementations / instances under aggregation.
//
// Given those, and that any implementation *must* accept valid StrongFingerprints from 'thin air' anyway (or aggregation won't work),
// I'm choosing here to neuter the sidechannel altogether (this has the added housekeeping benefit of Cache.Core types not appearing on BuildXL.Engine.Cache interfaces at all).
StrongFingerprint reconstructedStrongFingerprint = new StrongFingerprint(
weak: new WeakFingerprintHash(new Hash(weakFingerprint.Hash)),
casElement: new CasHash(new global::BuildXL.Cache.Interfaces.Hash(pathSetHash)),
hashElement: new global::BuildXL.Cache.Interfaces.Hash(strongFingerprint.Hash),
cacheId: "Thin Air");
Possible <CasEntries, Failure> maybeEntry = await m_cache.GetCacheEntryAsync(reconstructedStrongFingerprint);
if (maybeEntry.Succeeded)
{
Contract.Assume(maybeEntry.Result != null, "Miss is supposed to be indicated with NoMatchingFingerprintFailure");
return(TryConvertCasEntriesToCacheEntry(maybeEntry.Result, null).Then <CacheEntry?>(e => e));
}
else
{
// TODO: This API will fail just because an entry isn't available, and that case isn't distinguishable (at least looking only at the interface).
// We can determine that case by reflecting here, in order to fit the requirements of the BuildXL-side interface; but that is quite fragile
// and requires some not-yet-prescribed co-operation from the implementations as to failure type. Instead, for a successful query (with no results),
// define a result type that indicates found / not-found (if only we had easy discriminated unions in C#!)
if (maybeEntry.Failure is NoMatchingFingerprintFailure)
{
return((CacheEntry?)null);
}
return(maybeEntry.Failure);
}
}
/// <inheritdoc />
public async Task <Possible <CacheEntryPublishResult, Failure> > TryPublishCacheEntryAsync(
WeakContentFingerprint weakFingerprint,
ContentHash pathSetHash,
StrongContentFingerprint strongFingerprint,
CacheEntry entry,
CacheEntryPublishMode publishMode = CacheEntryPublishMode.CreateNew,
PublishCacheEntryOptions options = default)
{
// We can request semantics appropriate for CreateNewOrReplaceExisting via CacheDeterminism.SinglePhaseNonDeterministic
// Note that conflict-rejections / failures may still occur.
CacheDeterminism determinism = publishMode == CacheEntryPublishMode.CreateNewOrReplaceExisting
? CacheDeterminism.SinglePhaseNonDeterministic
: default(CacheDeterminism);
// N.B. this includes the metadata hash.
CasEntries adaptedHashes = new CasEntries(
entry.ToArray(h => new CasHash(new global::BuildXL.Cache.Interfaces.Hash(h))),
determinism);
Possible <FullCacheRecordWithDeterminism, Failure> maybePublished = await PerformFingerprintCacheOperationAsync(
() => m_cache.AddOrGetAsync(
weak: new WeakFingerprintHash(new Hash(weakFingerprint.Hash)),
casElement: new CasHash(new Hash(pathSetHash)),
hashElement: new Hash(strongFingerprint.Hash),
hashes: adaptedHashes,
urgencyHint: options.ShouldPublishAssociatedContent ? UrgencyHint.RegisterAssociatedContent : UrgencyHint.SkipRegisterContent),
nameof(TryPublishCacheEntryAsync));
if (maybePublished.Succeeded)
{
if (maybePublished.Result.Record == null)
{
// Happy path: Entry accepted without an alternative.
return(CacheEntryPublishResult.CreatePublishedResult());
}
else
{
// Less happy path: The underlying store has an alternative entry that we need to use instead.
Possible <CacheEntry, Failure> maybeConvertedConflictingEntry = TryConvertCasEntriesToCacheEntry(maybePublished.Result.Record.CasEntries, maybePublished.Result.Record.CacheId);
if (maybeConvertedConflictingEntry.Succeeded)
{
return(CacheEntryPublishResult.CreateConflictResult(maybeConvertedConflictingEntry.Result));
}
else
{
return(maybeConvertedConflictingEntry.Failure.Annotate(
"The cache returned a conflicting entry (rejecting the proposed entry), but the conflicting entry is invalid."));
}
}
}
else
{
return(maybePublished.Failure);
}
}
/// <nodoc />
public TwoPhaseCachingInfo(
WeakContentFingerprint weakFingerprint,
ContentHash pathSetHash,
StrongContentFingerprint strongFingerprint,
CacheEntry cacheEntry)
{
WeakFingerprint = weakFingerprint;
PathSetHash = pathSetHash;
StrongFingerprint = strongFingerprint;
CacheEntry = cacheEntry;
}
public void StrongFingerprintVariations()
{
PathTable pathTable = new PathTable();
AbsolutePath pathA = AbsolutePath.Create(pathTable, X("/X/bar"));
AbsolutePath pathB = AbsolutePath.Create(pathTable, X("/X/foo"));
WeakContentFingerprint wfp1 = new WeakContentFingerprint(CreateFakeFingerprint(1));
WeakContentFingerprint wfp2 = new WeakContentFingerprint(CreateFakeFingerprint(2));
ContentHash content1 = CreateFakeContentHash(3);
ContentHash content2 = CreateFakeContentHash(4);
FingerprintingHarness harness = new FingerprintingHarness(pathTable);
// Initial
harness.AssertFingerprintIsNew(
wfp1,
ObservedInput.CreateFileContentRead(pathA, content1));
// Change WFP of Initial
harness.AssertFingerprintIsNew(
wfp2, // Changed
ObservedInput.CreateFileContentRead(pathA, content1));
// Change observation type of Initial
harness.AssertFingerprintIsNew(
wfp1,
ObservedInput.CreateAbsentPathProbe(pathA)); // Changed
harness.AssertFingerprintIsNew(
wfp1,
ObservedInput.CreateDirectoryEnumeration(pathA, new DirectoryFingerprint(content1))); // Changed
// Change file content of Initial
harness.AssertFingerprintIsNew(
wfp1,
ObservedInput.CreateFileContentRead(pathA, content2)); // Changed
// Add a second file to Initial
harness.AssertFingerprintIsNew(
wfp1,
ObservedInput.CreateFileContentRead(pathA, content1),
ObservedInput.CreateFileContentRead(pathB, content1)); // New
// Change WFP versus previous
harness.AssertFingerprintIsNew(
wfp2, // Changed
ObservedInput.CreateFileContentRead(pathA, content1),
ObservedInput.CreateFileContentRead(pathB, content1));
harness.AssertFingerprintIsNew(
wfp1,
ObservedInput.CreateExistingDirectoryProbe(pathA)); // Changed
}
/// <nodoc />
public static TwoPhaseCachingInfo Deserialize(BuildXLReader reader)
{
var buffer = new byte[s_maxContentHashFingerprintLength];
WeakContentFingerprint weakFingerprint = new WeakContentFingerprint(DeserializeFingerprint(reader, buffer));
ContentHash pathSetHash = DeserializeHash(reader, buffer);
StrongContentFingerprint strongFingerprint = new StrongContentFingerprint(DeserializeFingerprint(reader, buffer));
CacheEntry cacheEntry = DeserializeCacheEntry(reader, buffer);
return(new TwoPhaseCachingInfo(weakFingerprint, pathSetHash, strongFingerprint, cacheEntry));
}
/// <inheritdoc />
public Task <Possible <CacheEntryPublishResult, Failure> > TryPublishCacheEntryAsync(
WeakContentFingerprint weakFingerprint,
ContentHash pathSetHash,
StrongContentFingerprint strongFingerprint,
CacheEntry entry,
CacheEntryPublishMode mode = CacheEntryPublishMode.CreateNew,
PublishCacheEntryOptions options = default)
{
var newNode = new Node(pathSetHash, strongFingerprint, entry);
var updatedNode = m_entries.AddOrUpdate(
weakFingerprint,
newNode,
(key, node) => node,
(key, node, existingNode) =>
{
Node currentNode = existingNode;
Node priorNode = null;
while (currentNode != null)
{
if (node.PathSetHash == currentNode.PathSetHash && node.StrongFingerprint == currentNode.StrongFingerprint)
{
if (mode == CacheEntryPublishMode.CreateNewOrReplaceExisting)
{
if (priorNode != null)
{
priorNode.Next = node;
}
node.Next = node;
}
return(existingNode);
}
priorNode = currentNode;
currentNode = currentNode.Next;
}
node.Next = existingNode;
return(node);
}).Item.Value;
if (mode == CacheEntryPublishMode.CreateNew)
{
if (updatedNode != newNode)
{
return(Task.FromResult(new Possible <CacheEntryPublishResult, Failure>(CacheEntryPublishResult.CreateConflictResult(updatedNode.CacheEntry))));
}
}
return(Task.FromResult(new Possible <CacheEntryPublishResult, Failure>(CacheEntryPublishResult.CreatePublishedResult())));
}
/// <summary>
/// Gets the cache descriptor metadata given its content hash
/// </summary>
/// <returns>the metadata, or a Failure{<see cref="PipFingerprintEntry"/>} if metadata was retrieved
/// but was a different kind, null if content was not available, or standard <see cref="Failure"/></returns>
public virtual async Task <Possible <PipCacheDescriptorV2Metadata> > TryRetrieveMetadataAsync(
Pip pip,
// TODO: Do we need these fingerprints given that the metadata hash is provided by this interface in the first place
WeakContentFingerprint weakFingerprint,
StrongContentFingerprint strongFingerprint,
ContentHash metadataHash,
ContentHash pathSetHash)
{
BoxRef <long> metadataSize = new BoxRef <long>();
Possible <PipFingerprintEntry> maybeMetadata =
await ArtifactContentCache.TryLoadAndDeserializeContentWithRetry <PipFingerprintEntry>(
LoggingContext,
metadataHash,
contentSize : metadataSize,
cancellationToken : Context.CancellationToken,
shouldRetry : possibleResult => !possibleResult.Succeeded || (possibleResult.Result != null && possibleResult.Result.IsCorrupted),
maxRetry : PipFingerprintEntry.LoadingAndDeserializingRetries);
if (!maybeMetadata.Succeeded)
{
return(maybeMetadata.Failure);
}
Counters.IncrementCounter(PipCachingCounter.LoadedMetadataCount);
Counters.AddToCounter(PipCachingCounter.LoadedMetadataSize, metadataSize.Value);
var metadataEntry = maybeMetadata.Result;
if (metadataEntry == null)
{
return((PipCacheDescriptorV2Metadata)null);
}
if (metadataEntry.Kind != PipFingerprintEntryKind.DescriptorV2)
{
// Metadata is incorrect kind.
var message = I($"Expected metadata kind is '{nameof(PipFingerprintEntryKind.DescriptorV2)}' but got '{metadataEntry.Kind}'");
return(new Failure <PipFingerprintEntry>(metadataEntry, new Failure <string>(message)));
}
return((PipCacheDescriptorV2Metadata)metadataEntry.Deserialize(
Context.CancellationToken,
new CacheQueryData
{
WeakContentFingerprint = weakFingerprint,
PathSetHash = pathSetHash,
StrongContentFingerprint = strongFingerprint,
MetadataHash = metadataHash,
ContentCache = ArtifactContentCache
}));
}
internal FingerprintStoreEntry CreateFingerprintStoreEntry(
Process pip,
PipFingerprintKeys pipFingerprintKeys,
WeakContentFingerprint weakFingerprint,
ProcessStrongFingerprintComputationData strongFingerprintData,
bool mustStorePathEntry = true,
bool cacheWeakFingerprintSerialization = false)
{
Counters.IncrementCounter(FingerprintStoreCounters.CreateFingerprintStoreEntryCount);
using (Counters.StartStopwatch(FingerprintStoreCounters.CreateFingerprintStoreEntryTime))
{
string pipSerializedWeakFingerprint = null;
if (cacheWeakFingerprintSerialization)
{
pipSerializedWeakFingerprint = m_weakFingerprintSerializationTransientCache.GetOrAdd(
pip.PipId,
(pipId, p) =>
{
Counters.IncrementCounter(FingerprintStoreCounters.JsonSerializationWeakFingerprintCount);
return(JsonSerialize(p));
},
pip);
}
else
{
if (!m_weakFingerprintSerializationTransientCache.TryRemove(pip.PipId, out pipSerializedWeakFingerprint))
{
Counters.IncrementCounter(FingerprintStoreCounters.JsonSerializationWeakFingerprintCount);
pipSerializedWeakFingerprint = JsonSerialize(pip);
}
}
return(new FingerprintStoreEntry
{
// { pip formatted semi stable hash : weak fingerprint, strong fingerprint, path set hash }
PipToFingerprintKeys = new PipKVP(pip.FormattedSemiStableHash, pipFingerprintKeys),
// { weak fingerprint hash : weak fingerprint inputs }
WeakFingerprintToInputs = new KVP(pipFingerprintKeys.WeakFingerprint, pipSerializedWeakFingerprint),
StrongFingerprintEntry = new StrongFingerprintEntry
{
// { strong fingerprint hash: strong fingerprint inputs }
StrongFingerprintToInputs = new KVP(pipFingerprintKeys.StrongFingerprint, JsonSerialize(weakFingerprint, strongFingerprintData.PathSetHash, strongFingerprintData.ObservedInputs)),
// { path set hash : path set inputs }
// If fingerprint comparison is enabled, the entry should contain the pathset json.
PathSetHashToInputs = mustStorePathEntry ? new KVP(pipFingerprintKeys.FormattedPathSetHash, JsonSerialize(strongFingerprintData)) : default,
private async Task PublishExpectingNoConflict(
WeakContentFingerprint weak,
ContentHash pathSetHash,
StrongContentFingerprint strong,
CacheEntry entry)
{
Possible <CacheEntryPublishResult> maybePublished = await FingerprintStore.TryPublishCacheEntryAsync(
weak,
pathSetHash,
strong,
entry);
XAssert.IsTrue(maybePublished.Succeeded);
CacheEntryPublishResult publishResult = maybePublished.Result;
XAssert.AreEqual(CacheEntryPublishStatus.Published, publishResult.Status);
}
/// <inheritdoc />
public Task <Possible <CacheEntry?, Failure> > TryGetCacheEntryAsync(WeakContentFingerprint weakFingerprint, ContentHash pathSetHash, StrongContentFingerprint strongFingerprint)
{
Node node;
if (m_entries.TryGetValue(weakFingerprint, out node))
{
while (node != null)
{
if (node.PathSetHash == pathSetHash && node.StrongFingerprint == strongFingerprint)
{
return(Task.FromResult(new Possible <CacheEntry?, Failure>(node.CacheEntry)));
}
node = node.Next;
}
}
return(Task.FromResult(new Possible <CacheEntry?, Failure>((CacheEntry?)null)));
}
internal FingerprintStoreEntry CreateFingerprintStoreEntry(
Process pip,
PipFingerprintKeys pipFingerprintKeys,
WeakContentFingerprint weakFingerprint,
ProcessStrongFingerprintComputationData strongFingerprintData,
bool mustStorePathEntry = true)
{
return(new FingerprintStoreEntry
{
// { pip formatted semi stable hash : weak fingerprint, strong fingerprint, path set hash }
PipToFingerprintKeys = new PipKVP(pip.FormattedSemiStableHash, pipFingerprintKeys),
// { weak fingerprint hash : weak fingerprint inputs }
WeakFingerprintToInputs = new KVP(pipFingerprintKeys.WeakFingerprint, JsonSerialize(pip)),
StrongFingerprintEntry = new StrongFingerprintEntry
{
// { strong fingerprint hash: strong fingerprint inputs }
StrongFingerprintToInputs = new KVP(pipFingerprintKeys.StrongFingerprint, JsonSerialize(weakFingerprint, strongFingerprintData.PathSetHash, strongFingerprintData.ObservedInputs)),
// { path set hash : path set inputs }
// If fingerprint comparison is enabled, the entry should contain the pathset json.
PathSetHashToInputs = mustStorePathEntry ? new KVP(pipFingerprintKeys.FormattedPathSetHash, JsonSerialize(strongFingerprintData)) : default,
/// <summary>
/// See <see cref="ITwoPhaseFingerprintStore.TryGetCacheEntryAsync"/>
/// </summary>
public virtual async Task <Possible <CacheEntry?, Failure> > TryGetCacheEntryAsync(
Pip pip,
WeakContentFingerprint weakFingerprint,
ContentHash pathSetHash,
StrongContentFingerprint strongFingerprint)
{
var result = await TwoPhaseFingerprintStore.TryGetCacheEntryAsync(weakFingerprint, pathSetHash, strongFingerprint);
if (result.Succeeded)
{
Tracing.Logger.Log.PipTwoPhaseCacheGetCacheEntry(
LoggingContext,
pip.GetDescription(Context),
weakFingerprint.ToString(),
pathSetHash.ToString(),
strongFingerprint.ToString(),
result.Result.HasValue ? result.Result.Value.MetadataHash.ToString() : "<NOVALUE>");
}
return(result);
}
public async Task ListMultipleEntriesForSamePathSet()
{
ContentHash pathSetHash = await AddContent("This is a list of paths");
WeakContentFingerprint weak = CreateWeakFingerprint("Fingerprint text here");
StrongContentFingerprint strongA = CreateStrongFingerprint("Strong fingerprint text here");
StrongContentFingerprint strongB = CreateStrongFingerprint("Slightly different fingerprint text here");
CacheEntry entryA = await CreateCacheEntryAndStoreContent("Metadata A", "A", "B");
CacheEntry entryB = await CreateCacheEntryAndStoreContent("Metadata B", "A-prime", "B-prime");
await PublishExpectingNoConflict(weak, pathSetHash, strongA, entryA);
await PublishExpectingNoConflict(weak, pathSetHash, strongB, entryB);
List <PublishedEntryRef> refs = new List <PublishedEntryRef>();
foreach (Task <Possible <PublishedEntryRef, Failure> > maybeEntryTask in FingerprintStore.ListPublishedEntriesByWeakFingerprint(weak))
{
Possible <PublishedEntryRef> maybeEntry = await maybeEntryTask;
XAssert.IsTrue(maybeEntry.Succeeded);
refs.Add(maybeEntry.Result);
}
if (refs[0].StrongFingerprint == strongA)
{
XAssert.AreEqual(strongB, refs[1].StrongFingerprint);
XAssert.AreEqual(pathSetHash, refs[1].PathSetHash);
}
else
{
XAssert.AreEqual(strongB, refs[0].StrongFingerprint);
XAssert.AreEqual(pathSetHash, refs[0].PathSetHash);
XAssert.AreEqual(strongA, refs[1].StrongFingerprint);
XAssert.AreEqual(pathSetHash, refs[1].PathSetHash);
}
}
public async Task FailedPublishReturnsConflictingEntry()
{
ContentHash pathSetHash = await AddContent("This is a list of paths");
WeakContentFingerprint weak = CreateWeakFingerprint("Fingerprint text here");
StrongContentFingerprint strong = CreateStrongFingerprint("Strong fingerprint text here");
CacheEntry originalEntry = await CreateCacheEntryAndStoreContent("Metadata", "A", "B");
Possible <CacheEntryPublishResult> maybePublished = await FingerprintStore.TryPublishCacheEntryAsync(
weak,
pathSetHash,
strong,
originalEntry);
XAssert.IsTrue(maybePublished.Succeeded);
CacheEntryPublishResult publishResult = maybePublished.Result;
XAssert.AreEqual(CacheEntryPublishStatus.Published, publishResult.Status);
CacheEntry successorEntry = await CreateCacheEntryAndStoreContent("Metadata", "Different A", "Different B");
Possible <CacheEntryPublishResult> maybePublishedAgain = await FingerprintStore.TryPublishCacheEntryAsync(
weak,
pathSetHash,
strong,
successorEntry);
// The conflict info is in the result (not a failure).
XAssert.IsTrue(maybePublishedAgain.Succeeded);
CacheEntryPublishResult publishAgainResult = maybePublishedAgain.Result;
XAssert.AreEqual(CacheEntryPublishStatus.RejectedDueToConflictingEntry, publishAgainResult.Status);
// Original entry should be returned.
AssertCacheEntriesEquivalent(publishAgainResult.ConflictingEntry, originalEntry);
}
/// <inheritdoc />
public IEnumerable <Task <Possible <PublishedEntryRef, Failure> > > ListPublishedEntriesByWeakFingerprint(WeakContentFingerprint weak)
{
WeakFingerprintHash cacheCoreWeakFingerprint = new WeakFingerprintHash(new Hash(weak.Hash));
// TODO: We assume that everything up until the first sentinel is local. This is fine for a simple 'vertical' aggregator
// of a local and remote cache, but isn't general.
PublishedEntryRefLocality currentLocality = PublishedEntryRefLocality.Local;
foreach (Task <Possible <StrongFingerprint, Failure> > entryPromise in m_cache.EnumerateStrongFingerprints(cacheCoreWeakFingerprint))
{
if (entryPromise.IsCompleted && entryPromise.Result.Succeeded && entryPromise.Result.Result is StrongFingerprintSentinel)
{
currentLocality = PublishedEntryRefLocality.Remote;
continue;
}
yield return(AdaptPublishedEntry(entryPromise, currentLocality));
}
}
/// <inheritdoc />
public IEnumerable <Task <Possible <PublishedEntryRef, Failure> > > ListPublishedEntriesByWeakFingerprint(WeakContentFingerprint weak)
{
return(Enumerable.Empty <Task <Possible <PublishedEntryRef, Failure> > >());
}
/// <inheritdoc />
public IEnumerable <Task <Possible <PublishedEntryRef, Failure> > > ListPublishedEntriesByWeakFingerprint(WeakContentFingerprint weak)
{
Node node;
if (m_entries.TryGetValue(weak, out node))
{
while (node != null)
{
yield return(Task.FromResult(
new Possible <PublishedEntryRef, Failure>(
new PublishedEntryRef(
node.PathSetHash,
node.StrongFingerprint,
m_cacheId,
PublishedEntryRefLocality.Local))));
node = node.Next;
}
}
}
/// <inheritdoc />
public Task <Possible <CacheEntry?, Failure> > TryGetCacheEntryAsync(WeakContentFingerprint weakFingerprint, ContentHash pathSetHash, StrongContentFingerprint strongFingerprint)
{
return(Task.FromResult(new Possible <CacheEntry?, Failure>(result: null)));
}
/// <summary>
/// See <see cref="ITwoPhaseFingerprintStore.ListPublishedEntriesByWeakFingerprint"/>
/// </summary>
public virtual IEnumerable <Task <Possible <PublishedEntryRef, Failure> > > ListPublishedEntriesByWeakFingerprint(OperationContext operationContext, WeakContentFingerprint weak)
{
IEnumerator <Task <Possible <PublishedEntryRef, Failure> > > enumerator;
using (operationContext.StartOperation(PipExecutorCounter.CacheQueryingWeakFingerprintDuration))
{
enumerator = TwoPhaseFingerprintStore.ListPublishedEntriesByWeakFingerprint(weak).GetEnumerator();
}
while (true)
{
Task <Possible <PublishedEntryRef, Failure> > current;
using (operationContext.StartOperation(PipExecutorCounter.CacheQueryingWeakFingerprintDuration))
{
if (enumerator.MoveNext())
{
current = enumerator.Current;
}
else
{
break;
}
}
yield return(current);
}
}
public async Task TestHistoricMetadataPathStringRoundtrip()
{
LoggingContext loggingContext = CreateLoggingContextForTest();
PipExecutionContext context;
HistoricMetadataCache cache = null;
var hmcFolderName = "hmc";
for (int i = 0; i < 3; i++)
{
CreateHistoricCache(loggingContext, hmcFolderName, out context, out cache, out var memoryArtifactCache);
var process1 = SchedulerTest.CreateDummyProcess(context, new PipId(1));
var process2 = SchedulerTest.CreateDummyProcess(context, new PipId(2));
var pathTable = context.PathTable;
// Add some random paths to ensure path table indices are different after loading
AbsolutePath.Create(pathTable, X("/H/aslj/sfas/832.stxt"));
AbsolutePath.Create(pathTable, X("/R/f/s/Historic"));
AbsolutePath.Create(pathTable, X("/M/hgf/sf4as/83afsd"));
AbsolutePath.Create(pathTable, X("/Z/bd/sfas/Cache"));
var abPath1 = AbsolutePath.Create(pathTable, X("/H/aslj/sfas/p1OUT.bin"));
var abPath2 = AbsolutePath.Create(pathTable, X("/H/aslj/sfas/P2.txt"));
var pathSet1 = ObservedPathSetTestUtilities.CreatePathSet(
pathTable,
X("/X/a/b/c"),
X("/X/d/e"),
X("/X/a/b/c/d"));
PipCacheDescriptorV2Metadata metadata1 =
new PipCacheDescriptorV2Metadata
{
StaticOutputHashes = new List <AbsolutePathFileMaterializationInfo>
{
new AbsolutePathFileMaterializationInfo
{
AbsolutePath = abPath1.GetName(pathTable).ToString(context.StringTable),
Info = new BondFileMaterializationInfo {
FileName = "p1OUT.bin"
}
}
}
};
var storedPathSet1 = await cache.TryStorePathSetAsync(pathSet1, preservePathCasing : false);
var storedMetadata1 = await cache.TryStoreMetadataAsync(metadata1);
var weakFingerprint1 = new WeakContentFingerprint(FingerprintUtilities.CreateRandom());
var strongFingerprint1 = new StrongContentFingerprint(FingerprintUtilities.CreateRandom());
var cacheEntry = new CacheEntry(storedMetadata1.Result, nameof(HistoricMetadataCacheTests), ArrayView <ContentHash> .Empty);
var publishedCacheEntry = await cache.TryPublishCacheEntryAsync(process1, weakFingerprint1, storedPathSet1.Result, strongFingerprint1, cacheEntry);
var pathSet2 = ObservedPathSetTestUtilities.CreatePathSet(
pathTable,
X("/F/a/y/c"),
X("/B/d/e"),
X("/G/a/z/c/d"),
X("/B/a/b/c"));
PipCacheDescriptorV2Metadata metadata2 =
new PipCacheDescriptorV2Metadata
{
StaticOutputHashes = new List <AbsolutePathFileMaterializationInfo>
{
new AbsolutePathFileMaterializationInfo
{
AbsolutePath = abPath2.ToString(pathTable),
Info = new BondFileMaterializationInfo {
FileName = abPath2.GetName(pathTable).ToString(context.StringTable)
}
}
},
DynamicOutputs = new List <List <RelativePathFileMaterializationInfo> >
{
new List <RelativePathFileMaterializationInfo>
{
new RelativePathFileMaterializationInfo
{
RelativePath = @"dir\P2Dynamic.txt",
Info = new BondFileMaterializationInfo {
FileName = "p2dynamic.txt"
}
},
new RelativePathFileMaterializationInfo
{
RelativePath = @"dir\P2dynout2.txt",
Info = new BondFileMaterializationInfo {
FileName = null
}
}
}
}
};
var storedPathSet2 = await cache.TryStorePathSetAsync(pathSet2, preservePathCasing : false);
var storedMetadata2 = await cache.TryStoreMetadataAsync(metadata2);
var cacheEntry2 = new CacheEntry(storedMetadata2.Result, nameof(HistoricMetadataCacheTests), ArrayView <ContentHash> .Empty);
var strongFingerprint2 = new StrongContentFingerprint(FingerprintUtilities.CreateRandom());
var publishedCacheEntry2 = await cache.TryPublishCacheEntryAsync(process1, weakFingerprint1, storedPathSet2.Result, strongFingerprint2, cacheEntry2);
await cache.CloseAsync();
memoryArtifactCache.Clear();
PipExecutionContext loadedContext;
HistoricMetadataCache loadedCache;
TaskSourceSlim <bool> loadCompletionSource = TaskSourceSlim.Create <bool>();
TaskSourceSlim <bool> loadCalled = TaskSourceSlim.Create <bool>();
BoxRef <bool> calledLoad = new BoxRef <bool>();
CreateHistoricCache(loggingContext, "hmc", out loadedContext, out loadedCache, out memoryArtifactCache, loadTask: async hmc =>
{
loadCalled.SetResult(true);
await loadCompletionSource.Task;
});
var operationContext = OperationContext.CreateUntracked(loggingContext);
var retrievePathSet1Task = loadedCache.TryRetrievePathSetAsync(operationContext, WeakContentFingerprint.Zero, storedPathSet1.Result);
var retrievdMetadata1Task = loadedCache.TryRetrieveMetadataAsync(
process1,
WeakContentFingerprint.Zero,
StrongContentFingerprint.Zero,
storedMetadata1.Result,
storedPathSet1.Result);
var getCacheEntry1Task = loadedCache.TryGetCacheEntryAsync(
process1,
weakFingerprint1,
storedPathSet1.Result,
strongFingerprint1);
Assert.False(retrievePathSet1Task.IsCompleted, "Before load task completes. TryRetrievePathSetAsync operations should block");
Assert.False(retrievdMetadata1Task.IsCompleted, "Before load task completes. TryRetrieveMetadataAsync operations should block");
Assert.False(getCacheEntry1Task.IsCompleted, "Before load task completes. TryGetCacheEntryAsync operations should block");
Assert.True(loadCalled.Task.Wait(TimeSpan.FromSeconds(10)) && loadCalled.Task.Result, "Load should have been called in as a result of querying");
loadCompletionSource.SetResult(true);
var maybeLoadedPathSet1 = await retrievePathSet1Task;
var maybeLoadedMetadata1 = await retrievdMetadata1Task;
var maybeLoadedCacheEntry1 = await getCacheEntry1Task;
Assert.Equal(storedMetadata1.Result, maybeLoadedCacheEntry1.Result.Value.MetadataHash);
var maybeLoadedPathSet2 = await loadedCache.TryRetrievePathSetAsync(operationContext, WeakContentFingerprint.Zero, storedPathSet2.Result);
var maybeLoadedMetadata2 = await loadedCache.TryRetrieveMetadataAsync(
process2,
WeakContentFingerprint.Zero,
StrongContentFingerprint.Zero,
storedMetadata2.Result,
storedPathSet2.Result);
AssertPathSetEquals(pathTable, pathSet1, loadedContext.PathTable, maybeLoadedPathSet1.Result);
AssertPathSetEquals(pathTable, pathSet2, loadedContext.PathTable, maybeLoadedPathSet2.Result);
AssertMetadataEquals(metadata1, maybeLoadedMetadata1.Result);
AssertMetadataEquals(metadata2, maybeLoadedMetadata2.Result);
await loadedCache.CloseAsync();
}
}
/// <summary>
/// Attempts to store a fingerprint entry; this is a compare-exchange operation, in which <paramref name="previousEntry"/>
/// must match what is currently stored (or must be <c>null</c> if no entry is currently stored).
/// This operation will fail if the previous entry doesn't match, or if the store could not normally proceed.
/// </summary>
public async Task <Possible <CacheEntryPublishResult, Failure> > TryStoreFingerprintEntryAsync(
ContentFingerprint fingerprint,
PipFingerprintEntry entry,
PipFingerprintEntry previousEntry = null,
bool replaceExisting = true,
CacheQueryData cacheQueryData = null)
{
Contract.Assume(entry != null);
Analysis.IgnoreArgument(previousEntry); // See class remarks; replace semantics are broken.
// We have in hand a PipFingerprintEntry which the underlyign m_twoPhaseStore does not understand.
// We will serialize it and store it to the CAS, and that CAS hash will be the stored entry's MetadataHash.
// See symmetric deserialization in TryGetFingerprintEntryAsync.
Possible <ContentHash, Failure> maybeStored = await m_contentCache.TrySerializeAndStoreContent(entry);
if (!maybeStored.Succeeded)
{
return(maybeStored.Failure.Annotate("Failed to store cache-entry metadata to the CAS"));
}
ContentHash metadataHash = maybeStored.Result;
// The metadata (single-phase entry) is stored, so now we can construct an entry that references it.
// From now on, 'twoPhaseEntry' will mean 'the entry we are actually storing in the two-phase store'.
// Meanwhile, like any implementation, we assume that the referenced content (e.g. output files)
// were stored by the caller already.
ContentHash[] twoPhaseReferencedHashes = entry.OutputContentHashes.Select(b => b.ToContentHash()).Where(hash => !hash.IsSpecialValue()).ToArray();
CacheEntry twoPhaseEntry = new CacheEntry(metadataHash, null, ArrayView <ContentHash> .FromArray(twoPhaseReferencedHashes));
StrongContentFingerprint dummyFingerprint = ComputeDummyStrongFingerprint(
m_pathTable,
fingerprint);
var weakFingerprint = new WeakContentFingerprint(fingerprint.Hash);
Possible <CacheEntryPublishResult, Failure> maybePublished = await m_twoPhaseStore.TryPublishCacheEntryAsync(
weakFingerprint : weakFingerprint,
pathSetHash : s_dummyPathSetHash,
strongFingerprint : dummyFingerprint,
entry : twoPhaseEntry,
mode : replaceExisting?CacheEntryPublishMode.CreateNewOrReplaceExisting : CacheEntryPublishMode.CreateNew);
if (cacheQueryData != null)
{
cacheQueryData.WeakContentFingerprint = weakFingerprint;
cacheQueryData.PathSetHash = s_dummyPathSetHash;
cacheQueryData.StrongContentFingerprint = dummyFingerprint;
cacheQueryData.ContentCache = m_contentCache;
}
if (maybePublished.Succeeded)
{
if (maybePublished.Result.Status == CacheEntryPublishStatus.Published ||
(!replaceExisting && maybePublished.Result.Status == CacheEntryPublishStatus.RejectedDueToConflictingEntry))
{
return(maybePublished.Result);
}
else
{
// ISinglePhaseFingerprintStore represents conflicts as failures.
return(new Failure <string>(
"Failed to publish a cache entry; the underlying two-phase store indicated an entry conflict (maybe it does not allow replacement of existing entries)."));
}
}
else
{
return(maybePublished.Failure);
}
}
