/// <summary> /// Attempts to retrieve a fingerprint entry. /// If the query is successful, a <see cref="PipFingerprintEntry"/> is returned (or null, if there is no current entry for the fingerprint). /// </summary> public async Task <Possible <PipFingerprintEntry, Failure> > TryGetFingerprintEntryAsync(ContentFingerprint fingerprint, CacheQueryData cacheQueryData = null) { StrongContentFingerprint dummyFingerprint = ComputeDummyStrongFingerprint( m_pathTable, fingerprint); var weakFingerprint = new WeakContentFingerprint(fingerprint.Hash); Possible <CacheEntry?, Failure> maybeEntry = await m_twoPhaseStore.TryGetCacheEntryAsync( weakFingerprint : weakFingerprint, pathSetHash : s_dummyPathSetHash, strongFingerprint : dummyFingerprint); if (!maybeEntry.Succeeded) { return(maybeEntry.Failure); } if (!maybeEntry.Result.HasValue) { // Real miss. return((PipFingerprintEntry)null); } if (cacheQueryData != null) { cacheQueryData.WeakContentFingerprint = weakFingerprint; cacheQueryData.PathSetHash = s_dummyPathSetHash; cacheQueryData.StrongContentFingerprint = dummyFingerprint; cacheQueryData.MetadataHash = maybeEntry.Result.Value.MetadataHash; cacheQueryData.ContentCache = m_contentCache; } return(await TryLoadAndDeserializeContent(maybeEntry.Result.Value.MetadataHash)); }
/// <summary> /// See <see cref="ITwoPhaseFingerprintStore.TryPublishCacheEntryAsync"/> /// </summary> public virtual async Task <Possible <CacheEntryPublishResult, Failure> > TryPublishCacheEntryAsync( Pip pip, WeakContentFingerprint weakFingerprint, ContentHash pathSetHash, StrongContentFingerprint strongFingerprint, CacheEntry entry, CacheEntryPublishMode mode = CacheEntryPublishMode.CreateNew) { Contract.Requires(pathSetHash.IsValid); Contract.Requires(entry.MetadataHash.IsValid); var result = await TwoPhaseFingerprintStore.TryPublishCacheEntryAsync( weakFingerprint, pathSetHash, strongFingerprint, entry, mode); if (result.Succeeded) { var publishedEntry = result.Result.Status == CacheEntryPublishStatus.Published ? entry : result.Result.ConflictingEntry; Tracing.Logger.Log.PipTwoPhaseCachePublishCacheEntry( LoggingContext, pip.GetDescription(Context), weakFingerprint.ToString(), pathSetHash.ToString(), strongFingerprint.ToString(), entry.MetadataHash.ToString(), result.Result.Status.ToString(), publishedEntry.MetadataHash.ToString()); } return(result); }
public async Task RoundtripCacheEntryWithMetadata() { ContentHash pathSetHash = await AddContent("This is a list of paths"); WeakContentFingerprint weak = CreateWeakFingerprint("Fingerprint text here"); StrongContentFingerprint strong = CreateStrongFingerprint("Strong fingerprint text here"); CacheEntry entry = await CreateCacheEntryAndStoreContent("Metadata", "A", "B"); Possible <CacheEntryPublishResult> maybePublished = await FingerprintStore.TryPublishCacheEntryAsync( weak, pathSetHash, strong, entry); XAssert.IsTrue(maybePublished.Succeeded); CacheEntryPublishResult publishResult = maybePublished.Result; XAssert.AreEqual(CacheEntryPublishStatus.Published, publishResult.Status); Possible <CacheEntry?> maybeFetched = await FingerprintStore.TryGetCacheEntryAsync( weak, pathSetHash, strong); XAssert.IsTrue(maybeFetched.Succeeded); XAssert.IsTrue(maybeFetched.Result.HasValue, "Unexpected miss (was just stored)"); CacheEntry roundtrippedEntry = maybeFetched.Result.Value; AssertCacheEntriesEquivalent(entry, roundtrippedEntry); }
/// <inheritdoc /> public void DeserializeAndUpdate(BinaryLogReader.EventReader reader) { PipId = new PipId(reader.ReadUInt32Compact()); Kind = (FingerprintComputationKind)reader.ReadInt32Compact(); WeakFingerprint = reader.ReadWeakFingerprint(); StrongFingerprintComputations = reader.ReadReadOnlyList(r => new ProcessStrongFingerprintComputationData((BinaryLogReader.EventReader)r)); }
/// <summary> /// Helper functions for putting entries into the fingerprint store for sub-components. /// { pip formatted semi stable hash : weak fingerprint, strong fingerprint, path set hash } /// { weak fingerprint hash : weak fingerprint inputs } /// { strong fingerprint hash : strong fingerprint inputs } /// { path set hash : path set inputs } /// </summary> private FingerprintStoreEntry CreateAndStoreFingerprintStoreEntry( FingerprintStore fingerprintStore, Process pip, PipFingerprintKeys pipFingerprintKeys, WeakContentFingerprint weakFingerprint, ProcessStrongFingerprintComputationData strongFingerprintData, bool cacheWeakFingerprintSerialization = false) { // If we got this far, a new pip is being put in the store Counters.IncrementCounter(FingerprintStoreCounters.NumPipFingerprintEntriesPut); UpdateOrStorePipUniqueOutputHashEntry(fingerprintStore, pip); // A content hash-keyed entry will have the same value as long as the key is the same, so overwriting it is unnecessary var mustStorePathEntry = !fingerprintStore.ContainsContentHash(pipFingerprintKeys.FormattedPathSetHash) || CacheMissAnalysisEnabled; var entry = CreateFingerprintStoreEntry( pip, pipFingerprintKeys, weakFingerprint, strongFingerprintData, mustStorePathEntry: mustStorePathEntry, cacheWeakFingerprintSerialization: cacheWeakFingerprintSerialization); fingerprintStore.PutFingerprintStoreEntry(entry, storePathSet: mustStorePathEntry); return(entry); }
/// <summary> /// Gets the deserialized path set given its content hash /// </summary> public virtual async Task <Possible <ObservedPathSet> > TryRetrievePathSetAsync( OperationContext operationContext, // TODO: Do we need this fingerprint given that the path set hash is provided by this interface in the first place WeakContentFingerprint weakFingerprint, ContentHash pathSetHash) { using (operationContext.StartOperation(PipExecutorCounter.TryLoadPathSetFromContentCacheDuration)) { Possible <Stream> maybePathSetStream = await TryLoadAndOpenPathSetStreamAsync(pathSetHash); if (!maybePathSetStream.Succeeded) { return(maybePathSetStream.Failure); } using (operationContext.StartOperation(PipExecutorCounter.TryLoadPathSetFromContentCacheDeserializeDuration)) using (var pathSetReader = new BuildXLReader(debug: false, stream: maybePathSetStream.Result, leaveOpen: false)) { var maybeDeserialized = ObservedPathSet.TryDeserialize(PathTable, pathSetReader, m_pathExpander); if (!maybeDeserialized.Succeeded) { return(maybeDeserialized.Failure); } return(maybeDeserialized.Result); } } }
public void AssertFingerprintIsNew(WeakContentFingerprint weakFingerprint, params ObservedInput[] inputs) { var result = CreateResult(PathTable, inputs); StrongContentFingerprint fp = result.ComputeStrongFingerprint(PathTable, weakFingerprint, ContentHashingUtilities.ZeroHash); XAssert.IsFalse(Fingerprints.Contains(fp), "Duplicate strong fingerprint"); Fingerprints.Add(fp); }
/// <inheritdoc /> public Task <Possible <CacheEntryPublishResult, Failure> > TryPublishCacheEntryAsync( WeakContentFingerprint weakFingerprint, ContentHash pathSetHash, StrongContentFingerprint strongFingerprint, CacheEntry entry, CacheEntryPublishMode mode = CacheEntryPublishMode.CreateNew, PublishCacheEntryOptions options = default) { return(Task.FromResult(new Possible <CacheEntryPublishResult, Failure>(CacheEntryPublishResult.CreatePublishedResult()))); }
private static (WeakContentFingerprint wf, StrongContentFingerprint sf) GetBuildManifestHashKey(ContentHash hash) { var hashBytes = hash.ToByteArray(); Array.Resize(ref hashBytes, FingerprintUtilities.FingerprintLength); var wf = new WeakContentFingerprint(FingerprintUtilities.CreateFrom(hashBytes)); var sf = new StrongContentFingerprint(wf.Hash); return(wf, sf); }
/// <inheritdoc /> public async Task <Possible <CacheEntry?, Failure> > TryGetCacheEntryAsync( WeakContentFingerprint weakFingerprint, ContentHash pathSetHash, StrongContentFingerprint strongFingerprint) { // TODO: We need a different side channel for prefetching etc. other than strong fingerprint subclasses. // - Given aggregation of multiple *closely aligned* stores as the common case, // one has the problem of optimizing which instance you hand to which store (and if you get it wrong, maybe things are just slower). // - For it to work, we have to be rather disingenuous about equality (see examples below). // - Usage this way is extremely error prone (with a perf pit if you get it wrong: // // foreach (StrongFingerprint candidate in Enumerate(...)) { // StrongFingerprint satisifiable = ComputeAvailable(LoadPathSet(candidate.CasElement)); // if (satisfiable == candidate) { // TODO: We have to lie about equality! // GetCacheEntryAndRunPipFromCache(satisfiable); // TODO: Oops. Accidentally slower? // } // } // // - By itself, lying about equality (for sake of the side-channel) probably has unintended consequences: // FancyStrongFingerprintCache.TryGetValue(pathSetHash, out strongFingerprint); // TODO: Oops. Side channel is broken again. // // - The implementations have to figure out which StrongFingerprints have side channel info for them. This requires // reflecting at least; but implementations must be vigilant to StrongFingerprints from *other instances*. This can break invariants, // (e.g. what if someone had a dictionary of 'fingerprints I totally returned' to some other data, and indexed blindly), and means we // have more to worry about w.r.t. cohabitation of implementations / instances under aggregation. // // Given those, and that any implementation *must* accept valid StrongFingerprints from 'thin air' anyway (or aggregation won't work), // I'm choosing here to neuter the sidechannel altogether (this has the added housekeeping benefit of Cache.Core types not appearing on BuildXL.Engine.Cache interfaces at all). StrongFingerprint reconstructedStrongFingerprint = new StrongFingerprint( weak: new WeakFingerprintHash(new Hash(weakFingerprint.Hash)), casElement: new CasHash(new global::BuildXL.Cache.Interfaces.Hash(pathSetHash)), hashElement: new global::BuildXL.Cache.Interfaces.Hash(strongFingerprint.Hash), cacheId: "Thin Air"); Possible <CasEntries, Failure> maybeEntry = await m_cache.GetCacheEntryAsync(reconstructedStrongFingerprint); if (maybeEntry.Succeeded) { Contract.Assume(maybeEntry.Result != null, "Miss is supposed to be indicated with NoMatchingFingerprintFailure"); return(TryConvertCasEntriesToCacheEntry(maybeEntry.Result, null).Then <CacheEntry?>(e => e)); } else { // TODO: This API will fail just because an entry isn't available, and that case isn't distinguishable (at least looking only at the interface). // We can determine that case by reflecting here, in order to fit the requirements of the BuildXL-side interface; but that is quite fragile // and requires some not-yet-prescribed co-operation from the implementations as to failure type. Instead, for a successful query (with no results), // define a result type that indicates found / not-found (if only we had easy discriminated unions in C#!) if (maybeEntry.Failure is NoMatchingFingerprintFailure) { return((CacheEntry?)null); } return(maybeEntry.Failure); } }
/// <inheritdoc /> public async Task <Possible <CacheEntryPublishResult, Failure> > TryPublishCacheEntryAsync( WeakContentFingerprint weakFingerprint, ContentHash pathSetHash, StrongContentFingerprint strongFingerprint, CacheEntry entry, CacheEntryPublishMode publishMode = CacheEntryPublishMode.CreateNew, PublishCacheEntryOptions options = default) { // We can request semantics appropriate for CreateNewOrReplaceExisting via CacheDeterminism.SinglePhaseNonDeterministic // Note that conflict-rejections / failures may still occur. CacheDeterminism determinism = publishMode == CacheEntryPublishMode.CreateNewOrReplaceExisting ? CacheDeterminism.SinglePhaseNonDeterministic : default(CacheDeterminism); // N.B. this includes the metadata hash. CasEntries adaptedHashes = new CasEntries( entry.ToArray(h => new CasHash(new global::BuildXL.Cache.Interfaces.Hash(h))), determinism); Possible <FullCacheRecordWithDeterminism, Failure> maybePublished = await PerformFingerprintCacheOperationAsync( () => m_cache.AddOrGetAsync( weak: new WeakFingerprintHash(new Hash(weakFingerprint.Hash)), casElement: new CasHash(new Hash(pathSetHash)), hashElement: new Hash(strongFingerprint.Hash), hashes: adaptedHashes, urgencyHint: options.ShouldPublishAssociatedContent ? UrgencyHint.RegisterAssociatedContent : UrgencyHint.SkipRegisterContent), nameof(TryPublishCacheEntryAsync)); if (maybePublished.Succeeded) { if (maybePublished.Result.Record == null) { // Happy path: Entry accepted without an alternative. return(CacheEntryPublishResult.CreatePublishedResult()); } else { // Less happy path: The underlying store has an alternative entry that we need to use instead. Possible <CacheEntry, Failure> maybeConvertedConflictingEntry = TryConvertCasEntriesToCacheEntry(maybePublished.Result.Record.CasEntries, maybePublished.Result.Record.CacheId); if (maybeConvertedConflictingEntry.Succeeded) { return(CacheEntryPublishResult.CreateConflictResult(maybeConvertedConflictingEntry.Result)); } else { return(maybeConvertedConflictingEntry.Failure.Annotate( "The cache returned a conflicting entry (rejecting the proposed entry), but the conflicting entry is invalid.")); } } } else { return(maybePublished.Failure); } }
/// <nodoc /> public TwoPhaseCachingInfo( WeakContentFingerprint weakFingerprint, ContentHash pathSetHash, StrongContentFingerprint strongFingerprint, CacheEntry cacheEntry) { WeakFingerprint = weakFingerprint; PathSetHash = pathSetHash; StrongFingerprint = strongFingerprint; CacheEntry = cacheEntry; }
public void StrongFingerprintVariations() { PathTable pathTable = new PathTable(); AbsolutePath pathA = AbsolutePath.Create(pathTable, X("/X/bar")); AbsolutePath pathB = AbsolutePath.Create(pathTable, X("/X/foo")); WeakContentFingerprint wfp1 = new WeakContentFingerprint(CreateFakeFingerprint(1)); WeakContentFingerprint wfp2 = new WeakContentFingerprint(CreateFakeFingerprint(2)); ContentHash content1 = CreateFakeContentHash(3); ContentHash content2 = CreateFakeContentHash(4); FingerprintingHarness harness = new FingerprintingHarness(pathTable); // Initial harness.AssertFingerprintIsNew( wfp1, ObservedInput.CreateFileContentRead(pathA, content1)); // Change WFP of Initial harness.AssertFingerprintIsNew( wfp2, // Changed ObservedInput.CreateFileContentRead(pathA, content1)); // Change observation type of Initial harness.AssertFingerprintIsNew( wfp1, ObservedInput.CreateAbsentPathProbe(pathA)); // Changed harness.AssertFingerprintIsNew( wfp1, ObservedInput.CreateDirectoryEnumeration(pathA, new DirectoryFingerprint(content1))); // Changed // Change file content of Initial harness.AssertFingerprintIsNew( wfp1, ObservedInput.CreateFileContentRead(pathA, content2)); // Changed // Add a second file to Initial harness.AssertFingerprintIsNew( wfp1, ObservedInput.CreateFileContentRead(pathA, content1), ObservedInput.CreateFileContentRead(pathB, content1)); // New // Change WFP versus previous harness.AssertFingerprintIsNew( wfp2, // Changed ObservedInput.CreateFileContentRead(pathA, content1), ObservedInput.CreateFileContentRead(pathB, content1)); harness.AssertFingerprintIsNew( wfp1, ObservedInput.CreateExistingDirectoryProbe(pathA)); // Changed }
/// <nodoc /> public static TwoPhaseCachingInfo Deserialize(BuildXLReader reader) { var buffer = new byte[s_maxContentHashFingerprintLength]; WeakContentFingerprint weakFingerprint = new WeakContentFingerprint(DeserializeFingerprint(reader, buffer)); ContentHash pathSetHash = DeserializeHash(reader, buffer); StrongContentFingerprint strongFingerprint = new StrongContentFingerprint(DeserializeFingerprint(reader, buffer)); CacheEntry cacheEntry = DeserializeCacheEntry(reader, buffer); return(new TwoPhaseCachingInfo(weakFingerprint, pathSetHash, strongFingerprint, cacheEntry)); }
/// <inheritdoc /> public Task <Possible <CacheEntryPublishResult, Failure> > TryPublishCacheEntryAsync( WeakContentFingerprint weakFingerprint, ContentHash pathSetHash, StrongContentFingerprint strongFingerprint, CacheEntry entry, CacheEntryPublishMode mode = CacheEntryPublishMode.CreateNew, PublishCacheEntryOptions options = default) { var newNode = new Node(pathSetHash, strongFingerprint, entry); var updatedNode = m_entries.AddOrUpdate( weakFingerprint, newNode, (key, node) => node, (key, node, existingNode) => { Node currentNode = existingNode; Node priorNode = null; while (currentNode != null) { if (node.PathSetHash == currentNode.PathSetHash && node.StrongFingerprint == currentNode.StrongFingerprint) { if (mode == CacheEntryPublishMode.CreateNewOrReplaceExisting) { if (priorNode != null) { priorNode.Next = node; } node.Next = node; } return(existingNode); } priorNode = currentNode; currentNode = currentNode.Next; } node.Next = existingNode; return(node); }).Item.Value; if (mode == CacheEntryPublishMode.CreateNew) { if (updatedNode != newNode) { return(Task.FromResult(new Possible <CacheEntryPublishResult, Failure>(CacheEntryPublishResult.CreateConflictResult(updatedNode.CacheEntry)))); } } return(Task.FromResult(new Possible <CacheEntryPublishResult, Failure>(CacheEntryPublishResult.CreatePublishedResult()))); }
/// <summary> /// Gets the cache descriptor metadata given its content hash /// </summary> /// <returns>the metadata, or a Failure{<see cref="PipFingerprintEntry"/>} if metadata was retrieved /// but was a different kind, null if content was not available, or standard <see cref="Failure"/></returns> public virtual async Task <Possible <PipCacheDescriptorV2Metadata> > TryRetrieveMetadataAsync( Pip pip, // TODO: Do we need these fingerprints given that the metadata hash is provided by this interface in the first place WeakContentFingerprint weakFingerprint, StrongContentFingerprint strongFingerprint, ContentHash metadataHash, ContentHash pathSetHash) { BoxRef <long> metadataSize = new BoxRef <long>(); Possible <PipFingerprintEntry> maybeMetadata = await ArtifactContentCache.TryLoadAndDeserializeContentWithRetry <PipFingerprintEntry>( LoggingContext, metadataHash, contentSize : metadataSize, cancellationToken : Context.CancellationToken, shouldRetry : possibleResult => !possibleResult.Succeeded || (possibleResult.Result != null && possibleResult.Result.IsCorrupted), maxRetry : PipFingerprintEntry.LoadingAndDeserializingRetries); if (!maybeMetadata.Succeeded) { return(maybeMetadata.Failure); } Counters.IncrementCounter(PipCachingCounter.LoadedMetadataCount); Counters.AddToCounter(PipCachingCounter.LoadedMetadataSize, metadataSize.Value); var metadataEntry = maybeMetadata.Result; if (metadataEntry == null) { return((PipCacheDescriptorV2Metadata)null); } if (metadataEntry.Kind != PipFingerprintEntryKind.DescriptorV2) { // Metadata is incorrect kind. var message = I($"Expected metadata kind is '{nameof(PipFingerprintEntryKind.DescriptorV2)}' but got '{metadataEntry.Kind}'"); return(new Failure <PipFingerprintEntry>(metadataEntry, new Failure <string>(message))); } return((PipCacheDescriptorV2Metadata)metadataEntry.Deserialize( Context.CancellationToken, new CacheQueryData { WeakContentFingerprint = weakFingerprint, PathSetHash = pathSetHash, StrongContentFingerprint = strongFingerprint, MetadataHash = metadataHash, ContentCache = ArtifactContentCache })); }
internal FingerprintStoreEntry CreateFingerprintStoreEntry( Process pip, PipFingerprintKeys pipFingerprintKeys, WeakContentFingerprint weakFingerprint, ProcessStrongFingerprintComputationData strongFingerprintData, bool mustStorePathEntry = true, bool cacheWeakFingerprintSerialization = false) { Counters.IncrementCounter(FingerprintStoreCounters.CreateFingerprintStoreEntryCount); using (Counters.StartStopwatch(FingerprintStoreCounters.CreateFingerprintStoreEntryTime)) { string pipSerializedWeakFingerprint = null; if (cacheWeakFingerprintSerialization) { pipSerializedWeakFingerprint = m_weakFingerprintSerializationTransientCache.GetOrAdd( pip.PipId, (pipId, p) => { Counters.IncrementCounter(FingerprintStoreCounters.JsonSerializationWeakFingerprintCount); return(JsonSerialize(p)); }, pip); } else { if (!m_weakFingerprintSerializationTransientCache.TryRemove(pip.PipId, out pipSerializedWeakFingerprint)) { Counters.IncrementCounter(FingerprintStoreCounters.JsonSerializationWeakFingerprintCount); pipSerializedWeakFingerprint = JsonSerialize(pip); } } return(new FingerprintStoreEntry { // { pip formatted semi stable hash : weak fingerprint, strong fingerprint, path set hash } PipToFingerprintKeys = new PipKVP(pip.FormattedSemiStableHash, pipFingerprintKeys), // { weak fingerprint hash : weak fingerprint inputs } WeakFingerprintToInputs = new KVP(pipFingerprintKeys.WeakFingerprint, pipSerializedWeakFingerprint), StrongFingerprintEntry = new StrongFingerprintEntry { // { strong fingerprint hash: strong fingerprint inputs } StrongFingerprintToInputs = new KVP(pipFingerprintKeys.StrongFingerprint, JsonSerialize(weakFingerprint, strongFingerprintData.PathSetHash, strongFingerprintData.ObservedInputs)), // { path set hash : path set inputs } // If fingerprint comparison is enabled, the entry should contain the pathset json. PathSetHashToInputs = mustStorePathEntry ? new KVP(pipFingerprintKeys.FormattedPathSetHash, JsonSerialize(strongFingerprintData)) : default,
private async Task PublishExpectingNoConflict( WeakContentFingerprint weak, ContentHash pathSetHash, StrongContentFingerprint strong, CacheEntry entry) { Possible <CacheEntryPublishResult> maybePublished = await FingerprintStore.TryPublishCacheEntryAsync( weak, pathSetHash, strong, entry); XAssert.IsTrue(maybePublished.Succeeded); CacheEntryPublishResult publishResult = maybePublished.Result; XAssert.AreEqual(CacheEntryPublishStatus.Published, publishResult.Status); }
/// <inheritdoc /> public Task <Possible <CacheEntry?, Failure> > TryGetCacheEntryAsync(WeakContentFingerprint weakFingerprint, ContentHash pathSetHash, StrongContentFingerprint strongFingerprint) { Node node; if (m_entries.TryGetValue(weakFingerprint, out node)) { while (node != null) { if (node.PathSetHash == pathSetHash && node.StrongFingerprint == strongFingerprint) { return(Task.FromResult(new Possible <CacheEntry?, Failure>(node.CacheEntry))); } node = node.Next; } } return(Task.FromResult(new Possible <CacheEntry?, Failure>((CacheEntry?)null))); }
internal FingerprintStoreEntry CreateFingerprintStoreEntry( Process pip, PipFingerprintKeys pipFingerprintKeys, WeakContentFingerprint weakFingerprint, ProcessStrongFingerprintComputationData strongFingerprintData, bool mustStorePathEntry = true) { return(new FingerprintStoreEntry { // { pip formatted semi stable hash : weak fingerprint, strong fingerprint, path set hash } PipToFingerprintKeys = new PipKVP(pip.FormattedSemiStableHash, pipFingerprintKeys), // { weak fingerprint hash : weak fingerprint inputs } WeakFingerprintToInputs = new KVP(pipFingerprintKeys.WeakFingerprint, JsonSerialize(pip)), StrongFingerprintEntry = new StrongFingerprintEntry { // { strong fingerprint hash: strong fingerprint inputs } StrongFingerprintToInputs = new KVP(pipFingerprintKeys.StrongFingerprint, JsonSerialize(weakFingerprint, strongFingerprintData.PathSetHash, strongFingerprintData.ObservedInputs)), // { path set hash : path set inputs } // If fingerprint comparison is enabled, the entry should contain the pathset json. PathSetHashToInputs = mustStorePathEntry ? new KVP(pipFingerprintKeys.FormattedPathSetHash, JsonSerialize(strongFingerprintData)) : default,
/// <summary> /// See <see cref="ITwoPhaseFingerprintStore.TryGetCacheEntryAsync"/> /// </summary> public virtual async Task <Possible <CacheEntry?, Failure> > TryGetCacheEntryAsync( Pip pip, WeakContentFingerprint weakFingerprint, ContentHash pathSetHash, StrongContentFingerprint strongFingerprint) { var result = await TwoPhaseFingerprintStore.TryGetCacheEntryAsync(weakFingerprint, pathSetHash, strongFingerprint); if (result.Succeeded) { Tracing.Logger.Log.PipTwoPhaseCacheGetCacheEntry( LoggingContext, pip.GetDescription(Context), weakFingerprint.ToString(), pathSetHash.ToString(), strongFingerprint.ToString(), result.Result.HasValue ? result.Result.Value.MetadataHash.ToString() : "<NOVALUE>"); } return(result); }
public async Task ListMultipleEntriesForSamePathSet() { ContentHash pathSetHash = await AddContent("This is a list of paths"); WeakContentFingerprint weak = CreateWeakFingerprint("Fingerprint text here"); StrongContentFingerprint strongA = CreateStrongFingerprint("Strong fingerprint text here"); StrongContentFingerprint strongB = CreateStrongFingerprint("Slightly different fingerprint text here"); CacheEntry entryA = await CreateCacheEntryAndStoreContent("Metadata A", "A", "B"); CacheEntry entryB = await CreateCacheEntryAndStoreContent("Metadata B", "A-prime", "B-prime"); await PublishExpectingNoConflict(weak, pathSetHash, strongA, entryA); await PublishExpectingNoConflict(weak, pathSetHash, strongB, entryB); List <PublishedEntryRef> refs = new List <PublishedEntryRef>(); foreach (Task <Possible <PublishedEntryRef, Failure> > maybeEntryTask in FingerprintStore.ListPublishedEntriesByWeakFingerprint(weak)) { Possible <PublishedEntryRef> maybeEntry = await maybeEntryTask; XAssert.IsTrue(maybeEntry.Succeeded); refs.Add(maybeEntry.Result); } if (refs[0].StrongFingerprint == strongA) { XAssert.AreEqual(strongB, refs[1].StrongFingerprint); XAssert.AreEqual(pathSetHash, refs[1].PathSetHash); } else { XAssert.AreEqual(strongB, refs[0].StrongFingerprint); XAssert.AreEqual(pathSetHash, refs[0].PathSetHash); XAssert.AreEqual(strongA, refs[1].StrongFingerprint); XAssert.AreEqual(pathSetHash, refs[1].PathSetHash); } }
public async Task FailedPublishReturnsConflictingEntry() { ContentHash pathSetHash = await AddContent("This is a list of paths"); WeakContentFingerprint weak = CreateWeakFingerprint("Fingerprint text here"); StrongContentFingerprint strong = CreateStrongFingerprint("Strong fingerprint text here"); CacheEntry originalEntry = await CreateCacheEntryAndStoreContent("Metadata", "A", "B"); Possible <CacheEntryPublishResult> maybePublished = await FingerprintStore.TryPublishCacheEntryAsync( weak, pathSetHash, strong, originalEntry); XAssert.IsTrue(maybePublished.Succeeded); CacheEntryPublishResult publishResult = maybePublished.Result; XAssert.AreEqual(CacheEntryPublishStatus.Published, publishResult.Status); CacheEntry successorEntry = await CreateCacheEntryAndStoreContent("Metadata", "Different A", "Different B"); Possible <CacheEntryPublishResult> maybePublishedAgain = await FingerprintStore.TryPublishCacheEntryAsync( weak, pathSetHash, strong, successorEntry); // The conflict info is in the result (not a failure). XAssert.IsTrue(maybePublishedAgain.Succeeded); CacheEntryPublishResult publishAgainResult = maybePublishedAgain.Result; XAssert.AreEqual(CacheEntryPublishStatus.RejectedDueToConflictingEntry, publishAgainResult.Status); // Original entry should be returned. AssertCacheEntriesEquivalent(publishAgainResult.ConflictingEntry, originalEntry); }
/// <inheritdoc /> public IEnumerable <Task <Possible <PublishedEntryRef, Failure> > > ListPublishedEntriesByWeakFingerprint(WeakContentFingerprint weak) { WeakFingerprintHash cacheCoreWeakFingerprint = new WeakFingerprintHash(new Hash(weak.Hash)); // TODO: We assume that everything up until the first sentinel is local. This is fine for a simple 'vertical' aggregator // of a local and remote cache, but isn't general. PublishedEntryRefLocality currentLocality = PublishedEntryRefLocality.Local; foreach (Task <Possible <StrongFingerprint, Failure> > entryPromise in m_cache.EnumerateStrongFingerprints(cacheCoreWeakFingerprint)) { if (entryPromise.IsCompleted && entryPromise.Result.Succeeded && entryPromise.Result.Result is StrongFingerprintSentinel) { currentLocality = PublishedEntryRefLocality.Remote; continue; } yield return(AdaptPublishedEntry(entryPromise, currentLocality)); } }
/// <inheritdoc /> public IEnumerable <Task <Possible <PublishedEntryRef, Failure> > > ListPublishedEntriesByWeakFingerprint(WeakContentFingerprint weak) { return(Enumerable.Empty <Task <Possible <PublishedEntryRef, Failure> > >()); }
/// <inheritdoc /> public IEnumerable <Task <Possible <PublishedEntryRef, Failure> > > ListPublishedEntriesByWeakFingerprint(WeakContentFingerprint weak) { Node node; if (m_entries.TryGetValue(weak, out node)) { while (node != null) { yield return(Task.FromResult( new Possible <PublishedEntryRef, Failure>( new PublishedEntryRef( node.PathSetHash, node.StrongFingerprint, m_cacheId, PublishedEntryRefLocality.Local)))); node = node.Next; } } }
/// <inheritdoc /> public Task <Possible <CacheEntry?, Failure> > TryGetCacheEntryAsync(WeakContentFingerprint weakFingerprint, ContentHash pathSetHash, StrongContentFingerprint strongFingerprint) { return(Task.FromResult(new Possible <CacheEntry?, Failure>(result: null))); }
/// <summary> /// See <see cref="ITwoPhaseFingerprintStore.ListPublishedEntriesByWeakFingerprint"/> /// </summary> public virtual IEnumerable <Task <Possible <PublishedEntryRef, Failure> > > ListPublishedEntriesByWeakFingerprint(OperationContext operationContext, WeakContentFingerprint weak) { IEnumerator <Task <Possible <PublishedEntryRef, Failure> > > enumerator; using (operationContext.StartOperation(PipExecutorCounter.CacheQueryingWeakFingerprintDuration)) { enumerator = TwoPhaseFingerprintStore.ListPublishedEntriesByWeakFingerprint(weak).GetEnumerator(); } while (true) { Task <Possible <PublishedEntryRef, Failure> > current; using (operationContext.StartOperation(PipExecutorCounter.CacheQueryingWeakFingerprintDuration)) { if (enumerator.MoveNext()) { current = enumerator.Current; } else { break; } } yield return(current); } }
public async Task TestHistoricMetadataPathStringRoundtrip() { LoggingContext loggingContext = CreateLoggingContextForTest(); PipExecutionContext context; HistoricMetadataCache cache = null; var hmcFolderName = "hmc"; for (int i = 0; i < 3; i++) { CreateHistoricCache(loggingContext, hmcFolderName, out context, out cache, out var memoryArtifactCache); var process1 = SchedulerTest.CreateDummyProcess(context, new PipId(1)); var process2 = SchedulerTest.CreateDummyProcess(context, new PipId(2)); var pathTable = context.PathTable; // Add some random paths to ensure path table indices are different after loading AbsolutePath.Create(pathTable, X("/H/aslj/sfas/832.stxt")); AbsolutePath.Create(pathTable, X("/R/f/s/Historic")); AbsolutePath.Create(pathTable, X("/M/hgf/sf4as/83afsd")); AbsolutePath.Create(pathTable, X("/Z/bd/sfas/Cache")); var abPath1 = AbsolutePath.Create(pathTable, X("/H/aslj/sfas/p1OUT.bin")); var abPath2 = AbsolutePath.Create(pathTable, X("/H/aslj/sfas/P2.txt")); var pathSet1 = ObservedPathSetTestUtilities.CreatePathSet( pathTable, X("/X/a/b/c"), X("/X/d/e"), X("/X/a/b/c/d")); PipCacheDescriptorV2Metadata metadata1 = new PipCacheDescriptorV2Metadata { StaticOutputHashes = new List <AbsolutePathFileMaterializationInfo> { new AbsolutePathFileMaterializationInfo { AbsolutePath = abPath1.GetName(pathTable).ToString(context.StringTable), Info = new BondFileMaterializationInfo { FileName = "p1OUT.bin" } } } }; var storedPathSet1 = await cache.TryStorePathSetAsync(pathSet1, preservePathCasing : false); var storedMetadata1 = await cache.TryStoreMetadataAsync(metadata1); var weakFingerprint1 = new WeakContentFingerprint(FingerprintUtilities.CreateRandom()); var strongFingerprint1 = new StrongContentFingerprint(FingerprintUtilities.CreateRandom()); var cacheEntry = new CacheEntry(storedMetadata1.Result, nameof(HistoricMetadataCacheTests), ArrayView <ContentHash> .Empty); var publishedCacheEntry = await cache.TryPublishCacheEntryAsync(process1, weakFingerprint1, storedPathSet1.Result, strongFingerprint1, cacheEntry); var pathSet2 = ObservedPathSetTestUtilities.CreatePathSet( pathTable, X("/F/a/y/c"), X("/B/d/e"), X("/G/a/z/c/d"), X("/B/a/b/c")); PipCacheDescriptorV2Metadata metadata2 = new PipCacheDescriptorV2Metadata { StaticOutputHashes = new List <AbsolutePathFileMaterializationInfo> { new AbsolutePathFileMaterializationInfo { AbsolutePath = abPath2.ToString(pathTable), Info = new BondFileMaterializationInfo { FileName = abPath2.GetName(pathTable).ToString(context.StringTable) } } }, DynamicOutputs = new List <List <RelativePathFileMaterializationInfo> > { new List <RelativePathFileMaterializationInfo> { new RelativePathFileMaterializationInfo { RelativePath = @"dir\P2Dynamic.txt", Info = new BondFileMaterializationInfo { FileName = "p2dynamic.txt" } }, new RelativePathFileMaterializationInfo { RelativePath = @"dir\P2dynout2.txt", Info = new BondFileMaterializationInfo { FileName = null } } } } }; var storedPathSet2 = await cache.TryStorePathSetAsync(pathSet2, preservePathCasing : false); var storedMetadata2 = await cache.TryStoreMetadataAsync(metadata2); var cacheEntry2 = new CacheEntry(storedMetadata2.Result, nameof(HistoricMetadataCacheTests), ArrayView <ContentHash> .Empty); var strongFingerprint2 = new StrongContentFingerprint(FingerprintUtilities.CreateRandom()); var publishedCacheEntry2 = await cache.TryPublishCacheEntryAsync(process1, weakFingerprint1, storedPathSet2.Result, strongFingerprint2, cacheEntry2); await cache.CloseAsync(); memoryArtifactCache.Clear(); PipExecutionContext loadedContext; HistoricMetadataCache loadedCache; TaskSourceSlim <bool> loadCompletionSource = TaskSourceSlim.Create <bool>(); TaskSourceSlim <bool> loadCalled = TaskSourceSlim.Create <bool>(); BoxRef <bool> calledLoad = new BoxRef <bool>(); CreateHistoricCache(loggingContext, "hmc", out loadedContext, out loadedCache, out memoryArtifactCache, loadTask: async hmc => { loadCalled.SetResult(true); await loadCompletionSource.Task; }); var operationContext = OperationContext.CreateUntracked(loggingContext); var retrievePathSet1Task = loadedCache.TryRetrievePathSetAsync(operationContext, WeakContentFingerprint.Zero, storedPathSet1.Result); var retrievdMetadata1Task = loadedCache.TryRetrieveMetadataAsync( process1, WeakContentFingerprint.Zero, StrongContentFingerprint.Zero, storedMetadata1.Result, storedPathSet1.Result); var getCacheEntry1Task = loadedCache.TryGetCacheEntryAsync( process1, weakFingerprint1, storedPathSet1.Result, strongFingerprint1); Assert.False(retrievePathSet1Task.IsCompleted, "Before load task completes. TryRetrievePathSetAsync operations should block"); Assert.False(retrievdMetadata1Task.IsCompleted, "Before load task completes. TryRetrieveMetadataAsync operations should block"); Assert.False(getCacheEntry1Task.IsCompleted, "Before load task completes. TryGetCacheEntryAsync operations should block"); Assert.True(loadCalled.Task.Wait(TimeSpan.FromSeconds(10)) && loadCalled.Task.Result, "Load should have been called in as a result of querying"); loadCompletionSource.SetResult(true); var maybeLoadedPathSet1 = await retrievePathSet1Task; var maybeLoadedMetadata1 = await retrievdMetadata1Task; var maybeLoadedCacheEntry1 = await getCacheEntry1Task; Assert.Equal(storedMetadata1.Result, maybeLoadedCacheEntry1.Result.Value.MetadataHash); var maybeLoadedPathSet2 = await loadedCache.TryRetrievePathSetAsync(operationContext, WeakContentFingerprint.Zero, storedPathSet2.Result); var maybeLoadedMetadata2 = await loadedCache.TryRetrieveMetadataAsync( process2, WeakContentFingerprint.Zero, StrongContentFingerprint.Zero, storedMetadata2.Result, storedPathSet2.Result); AssertPathSetEquals(pathTable, pathSet1, loadedContext.PathTable, maybeLoadedPathSet1.Result); AssertPathSetEquals(pathTable, pathSet2, loadedContext.PathTable, maybeLoadedPathSet2.Result); AssertMetadataEquals(metadata1, maybeLoadedMetadata1.Result); AssertMetadataEquals(metadata2, maybeLoadedMetadata2.Result); await loadedCache.CloseAsync(); } }
/// <summary> /// Attempts to store a fingerprint entry; this is a compare-exchange operation, in which <paramref name="previousEntry"/> /// must match what is currently stored (or must be <c>null</c> if no entry is currently stored). /// This operation will fail if the previous entry doesn't match, or if the store could not normally proceed. /// </summary> public async Task <Possible <CacheEntryPublishResult, Failure> > TryStoreFingerprintEntryAsync( ContentFingerprint fingerprint, PipFingerprintEntry entry, PipFingerprintEntry previousEntry = null, bool replaceExisting = true, CacheQueryData cacheQueryData = null) { Contract.Assume(entry != null); Analysis.IgnoreArgument(previousEntry); // See class remarks; replace semantics are broken. // We have in hand a PipFingerprintEntry which the underlyign m_twoPhaseStore does not understand. // We will serialize it and store it to the CAS, and that CAS hash will be the stored entry's MetadataHash. // See symmetric deserialization in TryGetFingerprintEntryAsync. Possible <ContentHash, Failure> maybeStored = await m_contentCache.TrySerializeAndStoreContent(entry); if (!maybeStored.Succeeded) { return(maybeStored.Failure.Annotate("Failed to store cache-entry metadata to the CAS")); } ContentHash metadataHash = maybeStored.Result; // The metadata (single-phase entry) is stored, so now we can construct an entry that references it. // From now on, 'twoPhaseEntry' will mean 'the entry we are actually storing in the two-phase store'. // Meanwhile, like any implementation, we assume that the referenced content (e.g. output files) // were stored by the caller already. ContentHash[] twoPhaseReferencedHashes = entry.OutputContentHashes.Select(b => b.ToContentHash()).Where(hash => !hash.IsSpecialValue()).ToArray(); CacheEntry twoPhaseEntry = new CacheEntry(metadataHash, null, ArrayView <ContentHash> .FromArray(twoPhaseReferencedHashes)); StrongContentFingerprint dummyFingerprint = ComputeDummyStrongFingerprint( m_pathTable, fingerprint); var weakFingerprint = new WeakContentFingerprint(fingerprint.Hash); Possible <CacheEntryPublishResult, Failure> maybePublished = await m_twoPhaseStore.TryPublishCacheEntryAsync( weakFingerprint : weakFingerprint, pathSetHash : s_dummyPathSetHash, strongFingerprint : dummyFingerprint, entry : twoPhaseEntry, mode : replaceExisting?CacheEntryPublishMode.CreateNewOrReplaceExisting : CacheEntryPublishMode.CreateNew); if (cacheQueryData != null) { cacheQueryData.WeakContentFingerprint = weakFingerprint; cacheQueryData.PathSetHash = s_dummyPathSetHash; cacheQueryData.StrongContentFingerprint = dummyFingerprint; cacheQueryData.ContentCache = m_contentCache; } if (maybePublished.Succeeded) { if (maybePublished.Result.Status == CacheEntryPublishStatus.Published || (!replaceExisting && maybePublished.Result.Status == CacheEntryPublishStatus.RejectedDueToConflictingEntry)) { return(maybePublished.Result); } else { // ISinglePhaseFingerprintStore represents conflicts as failures. return(new Failure <string>( "Failed to publish a cache entry; the underlying two-phase store indicated an entry conflict (maybe it does not allow replacement of existing entries).")); } } else { return(maybePublished.Failure); } }