private static async Task <Possible <CacheEntry?, Failure> > TryGetTemporalCacheEntryAsync(
this ITwoPhaseFingerprintStore store,
LoggingContext loggingContext,
TimeTreeNode node,
ContentFingerprint fingerprint)
{
// Unblock the caller
await Task.Yield();
var result = await store.TryGetCacheEntryAsync(
new WeakContentFingerprint(fingerprint.Hash),
s_dummyPathSetHash,
node.NodeFingerprint);
if (result.Succeeded)
{
if (result.Result?.MetadataHash != null)
{
Logger.Log.TemporalCacheEntryTrace(loggingContext, I($"Successfully retrieved temporal cache entry: Node='{node}', Fingerprint='{fingerprint}' MetadataHash='{result.Result?.MetadataHash}'"));
}
else
{
Logger.Log.TemporalCacheEntryTrace(loggingContext, I($"Query for temporal cache entry returned no results: Node='{node}', Fingerprint='{fingerprint}'"));
}
}
else
{
Logger.Log.TemporalCacheEntryTrace(loggingContext, I($"Failed retrieval of temporal cache entry: Node='{node}', Fingerprint='{fingerprint}' Failure:='{result.Failure.DescribeIncludingInnerFailures()}'"));
}
return(result);
}
/// <summary>
/// Store the running time table to the cache
/// </summary>
public static async Task <Possible <Unit> > TryStoreRunningTimeTableAsync(
this EngineCache cache,
LoggingContext loggingContext,
string path,
PathTable pathTable,
ContentFingerprint performanceDataFingerprint,
DateTime?storeTime = null)
{
var absolutePath = AbsolutePath.Create(pathTable, path);
var storeResult = await cache.ArtifactContentCache.TryStoreAsync(
FileRealizationMode.Copy,
absolutePath.Expand(pathTable));
Logger.Log.HistoricPerfDataCacheTrace(loggingContext, I($"Storing running time table to cache: Success='{storeResult.Succeeded}'"));
if (!storeResult.Succeeded)
{
return(storeResult.Failure);
}
ContentHash hash = storeResult.Result;
var cacheEntry = new CacheEntry(hash, null, ArrayView <ContentHash> .Empty);
var publishResult = await cache.TwoPhaseFingerprintStore.TryPublishTemporalCacheEntryAsync(loggingContext, performanceDataFingerprint, cacheEntry, storeTime);
Logger.Log.HistoricPerfDataCacheTrace(loggingContext, I($"Publishing running time table from cache: Fingerprint='{performanceDataFingerprint}' Hash={hash}"));
return(publishResult);
}
public static async Task <Possible <Unit> > TryPublishTemporalCacheEntryAsync(
this ITwoPhaseFingerprintStore store,
LoggingContext loggingContext,
ContentFingerprint fingerprint,
CacheEntry entry,
DateTime?time = null)
{
// Attempt to store the cache entry into all the time buckets
var node = new TimeTreeNode(time);
var tasks = new List <Task <Possible <CacheEntryPublishResult, Failure> > >();
while (node != null)
{
tasks.Add(store.TryPublishNodeTemporalCacheEntryAsync(
loggingContext,
node,
fingerprint,
entry));
node = node.Parent;
}
await Task.WhenAll(tasks);
foreach (var task in tasks)
{
if (!task.Result.Succeeded)
{
return(task.Result.Failure);
}
}
return(Unit.Void);
}
/// <summary>
/// Attempts to get the cache entry which was last stored for the given fingerprint
/// </summary>
public static async Task <Possible <CacheEntry?> > TryGetLatestCacheEntryAsync(
this ITwoPhaseFingerprintStore store,
LoggingContext loggingContext,
ContentFingerprint fingerprint,
DateTime?time = null)
{
// This method starts at a leaf time node and traverses up until a parent node
// is found which has a corresponding cache entry. Then the children of that parent
// node are iterated recursively (latest in time first) to find the node that closest in time
// to the specified time parameter. This yields the cache entry added most recently.
var node = new TimeTreeNode(time);
while (node != null)
{
var getResult = await store.TryGetTemporalCacheEntryAsync(loggingContext, node, fingerprint);
if (!getResult.Succeeded)
{
return(getResult);
}
else if (getResult.Result != null)
{
return(await TryGetLatestChildCacheEntryAsync(loggingContext, node, store, fingerprint, getResult));
}
node = node.Parent;
}
return((CacheEntry?)null);
}
/// <summary>
/// Adds pip static fingerprint.
/// </summary>
/// <param name="pip">Pip.</param>
/// <param name="fingerprint">Static fingerprint.</param>
public void AddFingerprint(Pip pip, ContentFingerprint fingerprint)
{
Contract.Requires(pip != null);
m_staticFingerprintsToPips[fingerprint] = pip.PipId;
m_pipsToStaticFingerprints.Add(pip.PipId, fingerprint);
}
/// <summary>
/// Retrieve the running time table from the cache
/// </summary>
public static async Task <Possible <bool> > TryRetrieveRunningTimeTableAsync(
this EngineCache cache,
LoggingContext loggingContext,
string path,
PathTable pathTable,
ContentFingerprint graphSemistableFingerprint,
DateTime?time = null)
{
var possibleCacheEntry =
await cache.TwoPhaseFingerprintStore.TryGetLatestCacheEntryAsync(loggingContext, graphSemistableFingerprint, time);
if (!possibleCacheEntry.Succeeded)
{
Logger.Log.PerformanceDataCacheTrace(
loggingContext,
I($"Failed loading running time table entry from cache: Failure:{possibleCacheEntry.Failure.DescribeIncludingInnerFailures()}"));
return(possibleCacheEntry.Failure);
}
Logger.Log.PerformanceDataCacheTrace(
loggingContext,
I($"Loaded running time table entry from cache: Fingerprint='{graphSemistableFingerprint}' MetadataHash={possibleCacheEntry.Result?.MetadataHash ?? ContentHashingUtilities.ZeroHash}"));
if (!possibleCacheEntry.Result.HasValue)
{
return(false);
}
var runningTimeTableHash = possibleCacheEntry.Result.Value.MetadataHash;
var absolutePath = AbsolutePath.Create(pathTable, path);
var maybePinned = await cache.ArtifactContentCache.TryLoadAvailableContentAsync(new[] { runningTimeTableHash });
var result = await maybePinned.ThenAsync(
async pinResult =>
{
if (!pinResult.AllContentAvailable)
{
return(new Failure <string>(I($"Could not pin content for running time table '{runningTimeTableHash}'")));
}
return(await cache.ArtifactContentCache.TryMaterializeAsync(
FileRealizationMode.Copy,
absolutePath.Expand(pathTable),
runningTimeTableHash));
});
if (!result.Succeeded)
{
Logger.Log.PerformanceDataCacheTrace(
loggingContext,
I($"Failed loading running time table from cache: Failure:{result.Failure.DescribeIncludingInnerFailures()}"));
return(result.Failure);
}
Logger.Log.PerformanceDataCacheTrace(loggingContext, I($"Loaded running time table from cache: Path='{path}'"));
return(true);
}
private static async Task <Possible <CacheEntryPublishResult, Failure> > TryPublishNodeTemporalCacheEntryAsync(
this ITwoPhaseFingerprintStore store,
LoggingContext loggingContext,
TimeTreeNode node,
ContentFingerprint fingerprint,
CacheEntry entry)
{
// Unblock the caller
await Task.Yield();
var result = await store.TryPublishCacheEntryAsync(
new WeakContentFingerprint(fingerprint.Hash),
s_dummyPathSetHash,
node.NodeFingerprint,
entry);
Logger.Log.TemporalCacheEntryTrace(loggingContext, I($"Publishing temporal cache entry: Node='{node}', Fingerprint='{fingerprint}' MetadataHash='{entry.MetadataHash}' Success='{result.Succeeded}'"));
if (result.Succeeded)
{
Logger.Log.TemporalCacheEntryTrace(loggingContext, I($"Published temporal cache entry: Node='{node}', Fingerprint='{fingerprint}' Status='{result.Result.Status}'"));
}
else
{
Logger.Log.TemporalCacheEntryTrace(loggingContext, I($"Failed publish of temporal cache entry: Node='{node}', Fingerprint='{fingerprint}' Failure:='{result.Failure.DescribeIncludingInnerFailures()}'"));
}
return(result);
}
public static ContentFingerprint ComputeGraphSemistableFingerprint(
LoggingContext loggingContext,
PipTable pipTable,
PathTable pathTable)
{
var processSemistableHashes = pipTable.StableKeys
.Select(pipId => pipTable.GetMutable(pipId))
.Where(info => info.PipType == PipType.Process)
.Select(info => info.SemiStableHash)
.ToList();
processSemistableHashes.Sort();
var indicatorHashes = processSemistableHashes.Take(16).ToArray();
using (var hasher = new HashingHelper(pathTable, recordFingerprintString: false))
{
hasher.Add("Type", "GraphSemistableFingerprint");
foreach (var indicatorHash in indicatorHashes)
{
hasher.Add("IndicatorPipSemistableHash", indicatorHash);
}
var fingerprint = new ContentFingerprint(hasher.GenerateHash());
Logger.Log.PerformanceDataCacheTrace(loggingContext, I($"Computed graph semistable fingerprint: {fingerprint}"));
return(fingerprint);
}
}
public void TestPartialSealDirectoryMembersAffectFingerprints()
{
ContentFingerprint fingerprint1 = CreateFingerprintForPartialSealWithMember("f.txt");
ContentFingerprint fingerprint2 = CreateFingerprintForPartialSealWithMember("f.txt");
XAssert.AreEqual(fingerprint1, fingerprint2);
fingerprint1 = CreateFingerprintForPartialSealWithMember("f.txt");
fingerprint2 = CreateFingerprintForPartialSealWithMember("g.txt");
XAssert.AreNotEqual(fingerprint1, fingerprint2);
}
public void TestCompositeSharedOpaqueMembersAffectFingerprints()
{
ContentFingerprint fingerprint1 = CreateFingerprintForCompositeSharedOpaque(@"\\root", @"\\root\so1", @"\\root\so2");
ContentFingerprint fingerprint2 = CreateFingerprintForCompositeSharedOpaque(@"\\root", @"\\root\so1", @"\\root\so2");
XAssert.AreEqual(fingerprint1, fingerprint2);
fingerprint1 = CreateFingerprintForCompositeSharedOpaque(@"\\root", @"\\root\so1", @"\\root\so2");
fingerprint2 = CreateFingerprintForCompositeSharedOpaque(@"\\root", @"\\root\so1", @"\\root\so3");
XAssert.AreNotEqual(fingerprint1, fingerprint2);
}
/// <summary>
/// Computes a fingerprint for looking up fingerprint store
/// </summary>
private static ContentFingerprint ComputeFingerprint(
string key,
string environment)
{
using (var hasher = new BasicHashingHelper(recordFingerprintString: false))
{
hasher.Add("Type", "FingerprintStoreFingerprint");
hasher.Add("FormatVersion", FingerprintStore.FormatVersion.Version.ToString());
hasher.Add("LookupVersion", FingerprintStoreLookupVersion);
hasher.Add("Key", key);
hasher.Add("Environment", environment);
var fingerprint = new ContentFingerprint(hasher.GenerateHash());
return(fingerprint);
}
}
public static ContentFingerprint ComputePerformanceDataFingerprint(
LoggingContext loggingContext,
PathTable pathTable,
ContentFingerprint graphSemistableFingerprint,
string environmentFingerprint)
{
using (var hasher = new HashingHelper(pathTable, recordFingerprintString: false))
{
hasher.Add("Type", "PerformanceDataFingerprint");
hasher.Add("FormatVersion", HistoricPerfDataTable.FormatVersion);
hasher.Add("LookupVersion", PerformanceDataLookupVersion);
hasher.Add("GraphSemistableFingerprint", graphSemistableFingerprint.ToString());
hasher.Add("EnvironmentFingerprint", environmentFingerprint ?? string.Empty);
var fingerprint = new ContentFingerprint(hasher.GenerateHash());
Logger.Log.HistoricPerfDataCacheTrace(loggingContext, I($"Computed performance fingerprint: {fingerprint}"));
return(fingerprint);
}
}
/// <summary>
/// Computes a fingerprint for looking up fingerprint store
/// </summary>
private static ContentFingerprint ComputeFingerprint(
string key,
IConfiguration configuration)
{
var extraFingerprintSalts = new ExtraFingerprintSalts(
configuration,
configuration.Cache.CacheSalt,
null);
using (var hasher = new BasicHashingHelper(recordFingerprintString: false))
{
hasher.Add("Type", "FingerprintStoreFingerprint");
hasher.Add("FormatVersion", FingerprintStore.FormatVersion.Version.ToString());
hasher.Add("LookupVersion", FingerprintStoreLookupVersion);
hasher.Add("Key", key);
hasher.Add("FingerprintSalt", extraFingerprintSalts.FingerprintSalt);
var fingerprint = new ContentFingerprint(hasher.GenerateHash());
return(fingerprint);
}
}
/// <inheritdoc />
public override ContentFingerprint ComputeWeakFingerprint(Pip pip, out string fingerprintInputText)
{
ContentFingerprint fingerprint;
fingerprintInputText = string.Empty;
if (FingerprintTextEnabled)
{
// Force compute weak fingerprint if fingerprint text is requested.
fingerprint = base.ComputeWeakFingerprint(pip, out fingerprintInputText);
}
else if (pip.IndependentStaticFingerprint.Length > 0)
{
fingerprint = new ContentFingerprint(pip.IndependentStaticFingerprint);
}
else
{
fingerprint = base.ComputeWeakFingerprint(pip, out fingerprintInputText);
}
return(CompleteFingerprint(new PipWithFingerprint(pip, fingerprint, fingerprintInputText), out fingerprintInputText));
}
public static async Task <Possible <Unit> > TryPublishTemporalCacheEntryAsync(
this ITwoPhaseFingerprintStore store,
LoggingContext loggingContext,
ContentFingerprint fingerprint,
CacheEntry entry,
DateTime?time = null)
{
// Attempt to store the cache entry into all the time buckets
var node = new TimeTreeNode(time);
var tasks = new List <Task <Possible <CacheEntryPublishResult, Failure> > >();
while (node != null)
{
tasks.Add(store.TryPublishNodeTemporalCacheEntryAsync(
loggingContext,
node,
fingerprint,
entry));
// Don't store full cache entry for parent nodes since time tree addition
// will trigger pins (and consequently downloads) of content for conflicting entries
entry = new CacheEntry(entry.MetadataHash, entry.OriginatingCache, ArrayView <ContentHash> .Empty);
node = node.Parent;
}
await Task.WhenAll(tasks);
foreach (var task in tasks)
{
if (!task.Result.Succeeded)
{
return(task.Result.Failure);
}
}
return(Unit.Void);
}
public void TestCompositeSharedOpaqueFilterAffectsFingerprints()
{
ContentFingerprint fingerprint1 = CreateFingerprintForCompositeSharedOpaque(
@"\\root", new[] { @"\\root\so1" }, new SealDirectoryContentFilter(SealDirectoryContentFilter.ContentFilterKind.Include, ".*"));
ContentFingerprint fingerprint2 = CreateFingerprintForCompositeSharedOpaque(
@"\\root", new[] { @"\\root\so1" }, new SealDirectoryContentFilter(SealDirectoryContentFilter.ContentFilterKind.Include, ".*"));
XAssert.AreEqual(fingerprint1, fingerprint2);
fingerprint1 = CreateFingerprintForCompositeSharedOpaque(
@"\\root", new[] { @"\\root\so1" }, new SealDirectoryContentFilter(SealDirectoryContentFilter.ContentFilterKind.Include, ".*exe"));
fingerprint2 = CreateFingerprintForCompositeSharedOpaque(
@"\\root", new[] { @"\\root\so1" }, new SealDirectoryContentFilter(SealDirectoryContentFilter.ContentFilterKind.Include, ".*txt"));
XAssert.AreNotEqual(fingerprint1, fingerprint2);
fingerprint1 = CreateFingerprintForCompositeSharedOpaque(
@"\\root", new[] { @"\\root\so1" }, new SealDirectoryContentFilter(SealDirectoryContentFilter.ContentFilterKind.Include, ".*"));
fingerprint2 = CreateFingerprintForCompositeSharedOpaque(
@"\\root", new[] { @"\\root\so1" }, new SealDirectoryContentFilter(SealDirectoryContentFilter.ContentFilterKind.Exclude, ".*"));
XAssert.AreNotEqual(fingerprint1, fingerprint2);
}
private (Process process, ContentFingerprint contentFingerprint, string fingerprintText) GetFingerprintInfo(PipReference lazyPip)
{
// Make sure the extra event data has set the value properly here.
Contract.Requires(m_fingerprintSalts.HasValue, "m_fingerprintSalts is not set.");
Process pip = (Process)lazyPip.HydratePip();
string fingerprintText;
// This checks for missing content info for pips.
foreach (var dependency in pip.Dependencies)
{
if (!m_fileContentMap.ContainsKey(dependency))
{
return(pip, ContentFingerprint.Zero, "FINGERPRINT CONTAINS UNKNOWN DEPENDENCIES");
}
}
if (m_contentFingerprinter == null)
{
m_contentFingerprinter = new PipContentFingerprinter(
CachedGraph.Context.PathTable,
LookupHash,
m_fingerprintSalts.Value,
pathExpander: m_pathExpander,
pipDataLookup: CachedGraph.PipGraph.QueryFileArtifactPipData)
{
FingerprintTextEnabled = true,
};
}
// TODO: Allow specifying fingerprinting version on the command line
ContentFingerprint fingerprint = m_contentFingerprinter.ComputeWeakFingerprint(
pip,
out fingerprintText);
return(pip, fingerprint, fingerprintText);
}
/// <summary>
/// Attempts to get the cache entry for the child which comes latest in time
/// </summary>
private static async Task <Possible <CacheEntry?> > TryGetLatestChildCacheEntryAsync(
LoggingContext loggingContext,
TimeTreeNode node,
ITwoPhaseFingerprintStore store,
ContentFingerprint fingerprint,
Possible <CacheEntry?> fallbackResult)
{
var childNodes = node.Children;
var children = childNodes.Select(childNode => store.TryGetTemporalCacheEntryAsync(loggingContext, childNode, fingerprint)).ToList();
var childResults = await Task.WhenAll(children);
// In reverse order (i.e. later in time), check children to see which
// one has results
for (int i = childResults.Length - 1; i >= 0; i--)
{
var childResult = childResults[i];
var childNode = childNodes[i];
if (!childResult.Succeeded)
{
continue;
}
if (childResult.Result != null)
{
return(await TryGetLatestChildCacheEntryAsync(
loggingContext,
childNode,
store,
fingerprint,
childResult));
}
}
return(fallbackResult);
}
/// <summary>
/// Completes pip's independent static weak fingerprint with the fingerprints of pip's dependencies when appropriate.
/// </summary>
private ContentFingerprint CompleteFingerprint(PipWithFingerprint pipWithFingerprint, out string completeFingerprintText)
{
ContentFingerprint completeFingerprint = pipWithFingerprint.Fingerprint;
completeFingerprintText = pipWithFingerprint.FingerprintText ?? string.Empty;
Pip pip = pipWithFingerprint.Pip;
if (pip is Process process)
{
if (m_sealDirectoryFingerprintLookup != null)
{
using (var hasher = CreateHashingHelper(true))
{
hasher.Add("IndependentFingerprint", completeFingerprint.Hash);
hasher.AddOrderIndependentCollection(
"DirectoryDependencyFingerprints",
process.DirectoryDependencies,
(fp, d) => fp.Add(d.Path, m_sealDirectoryFingerprintLookup(d).Hash),
DirectoryComparer);
completeFingerprint = new ContentFingerprint(hasher.GenerateHash());
completeFingerprintText = FingerprintTextEnabled
? completeFingerprintText + Environment.NewLine + hasher.FingerprintInputText
: completeFingerprintText;
}
}
}
else if (pip is SealDirectory sealDirectory &&
sealDirectory.Kind == SealDirectoryKind.SharedOpaque &&
!sealDirectory.IsComposite)
{
if (!sealDirectory.IsComposite)
{
// For non-composite shared opaque directories, contents and composed directories are always empty, and therefore the static fingerprint
// is not strong enough, i.e. multiple shared opaques can share the same directory root. So in this case we need to add the fingerprint of the producer
if (m_directoryProducerFingerprintLookup != null)
{
DirectoryArtifact directory = sealDirectory.Directory;
using (var hasher = CreateHashingHelper(true))
{
hasher.Add("IndependentFingerprint", completeFingerprint.Hash);
hasher.Add(directory, m_directoryProducerFingerprintLookup(directory).Hash);
completeFingerprint = new ContentFingerprint(hasher.GenerateHash());
completeFingerprintText = FingerprintTextEnabled
? completeFingerprintText + Environment.NewLine + hasher.FingerprintInputText
: completeFingerprintText;
}
}
}
else if (sealDirectory.ComposedDirectories != null)
{
using (var hasher = CreateHashingHelper(true))
{
hasher.Add("IndependentFingerprint", completeFingerprint.Hash);
hasher.AddCollection <DirectoryArtifact, IReadOnlyList <DirectoryArtifact> >(
"ComposedDirectoryFingerprints",
sealDirectory.ComposedDirectories,
(fp, d) => fp.Add(d.Path, m_sealDirectoryFingerprintLookup(d).Hash));
completeFingerprint = new ContentFingerprint(hasher.GenerateHash());
completeFingerprintText = FingerprintTextEnabled
? completeFingerprintText + Environment.NewLine + hasher.FingerprintInputText
: completeFingerprintText;
}
}
}
return(completeFingerprint);
}
public override int Analyze()
{
List <PipReference> orderedPips = CachedGraph.PipGraph
.RetrievePipReferencesOfType(PipType.Process)
.Where(lazyPip => m_completedPips.Contains(lazyPip.PipId))
.OrderBy(lazyPip => CachedGraph.DirectedGraph.GetNodeHeight(lazyPip.PipId.ToNodeId()))
.ThenBy(lazyPip => lazyPip.SemiStableHash)
.ToList();
using (var fingerprintStream = File.Create(FingerprintFilePath, bufferSize: 64 << 10 /* 64 KB */))
{
using (
var fingerprintArchive = CompressFingerprintFile
? new ZipArchive(fingerprintStream, ZipArchiveMode.Create)
: null)
{
using (
var writer =
new StreamWriter(
CompressFingerprintFile
? fingerprintArchive.CreateEntry("fingerprint.txt", CompressionLevel.Fastest).Open()
: fingerprintStream))
{
var pipsAndFingerprintTexts = orderedPips.AsParallel().AsOrdered()
.WithDegreeOfParallelism(degreeOfParallelism: MaxDegreeOfParallelism)
.Select(p => GetFingerprintInfo(p));
int doneFingerprints = 0;
var t = new Timer(
o =>
{
var done = doneFingerprints;
Console.WriteLine("Fingerprints Done: {0} of {1}", done, orderedPips.Count);
},
null,
5000,
5000);
try
{
foreach (var pipAndFingerprintText in pipsAndFingerprintTexts)
{
doneFingerprints++;
Process pip = pipAndFingerprintText.process;
if (!IncludeServicePips && pip.IsStartOrShutdownKind)
{
continue;
}
ContentFingerprint fingerprint = pipAndFingerprintText.contentFingerprint;
string fingerprintText = pipAndFingerprintText.fingerprintText;
writer.WriteLine("PipStableId: Pip{0:X16}", pip.SemiStableHash);
writer.WriteLine(
"Pip Dependency Chain Length: {0}",
CachedGraph.DirectedGraph.GetNodeHeight(pip.PipId.ToNodeId()));
writer.WriteLine(pip.GetDescription(CachedGraph.Context));
writer.WriteLine("Fingerprint: {0}", fingerprint);
writer.WriteLine();
writer.WriteLine(fingerprintText);
writer.WriteLine();
writer.WriteLine();
}
}
finally
{
// kill and wait for the status timer to die...
using (var e = new AutoResetEvent(false))
{
t.Dispose(e);
e.WaitOne();
}
}
}
}
}
return(0);
}
/// <summary>
/// Gets pip's static fingerprints.
/// </summary>
/// <param name="pip">Pip.</param>
/// <param name="fingerprint">Static fingerprint.</param>
public bool TryGetFingerprint(Pip pip, out ContentFingerprint fingerprint)
{
Contract.Requires(pip != null);
return(TryGetFingerprint(pip.PipId, out fingerprint));
}
public EntryReader(SinglePhaseFingerprintStoreAdapter store, ContentFingerprint fingerprint)
{
Contract.Requires(store != null);
m_store = store;
m_fingerprint = fingerprint;
}
/// <summary>
/// Attempts to retrieve a fingerprint entry.
/// If the query is successful, a <see cref="PipFingerprintEntry"/> is returned (or null, if there is no current entry for the fingerprint).
/// </summary>
public async Task <Possible <PipFingerprintEntry, Failure> > TryGetFingerprintEntryAsync(ContentFingerprint fingerprint, CacheQueryData cacheQueryData = null)
{
StrongContentFingerprint dummyFingerprint = ComputeDummyStrongFingerprint(
m_pathTable,
fingerprint);
var weakFingerprint = new WeakContentFingerprint(fingerprint.Hash);
Possible <CacheEntry?, Failure> maybeEntry = await m_twoPhaseStore.TryGetCacheEntryAsync(
weakFingerprint : weakFingerprint,
pathSetHash : s_dummyPathSetHash,
strongFingerprint : dummyFingerprint);
if (!maybeEntry.Succeeded)
{
return(maybeEntry.Failure);
}
if (!maybeEntry.Result.HasValue)
{
// Real miss.
return((PipFingerprintEntry)null);
}
if (cacheQueryData != null)
{
cacheQueryData.WeakContentFingerprint = weakFingerprint;
cacheQueryData.PathSetHash = s_dummyPathSetHash;
cacheQueryData.StrongContentFingerprint = dummyFingerprint;
cacheQueryData.MetadataHash = maybeEntry.Result.Value.MetadataHash;
cacheQueryData.ContentCache = m_contentCache;
}
return(await TryLoadAndDeserializeContent(maybeEntry.Result.Value.MetadataHash));
}
private static StrongContentFingerprint ComputeDummyStrongFingerprint(PathTable pathTable, ContentFingerprint weakFingerprint)
{
using (var hasher = StrongContentFingerprint.CreateHashingHelper(
pathTable,
recordFingerprintString: false))
{
return(new StrongContentFingerprint(hasher.GenerateHash()));
}
}
/// <summary>
/// Returns a reader which can visit multiple fingerprint entries for a fingerprint (due to multiple cache levels, etc.)
/// </summary>
public IFingerprintEntryReader GetFingerprintEntryReader(ContentFingerprint fingerprint)
{
return(new EntryReader(this, fingerprint));
}
/// <summary>
/// Attempts to store a fingerprint entry; this is a compare-exchange operation, in which <paramref name="previousEntry"/>
/// must match what is currently stored (or must be <c>null</c> if no entry is currently stored).
/// This operation will fail if the previous entry doesn't match, or if the store could not normally proceed.
/// </summary>
public async Task <Possible <CacheEntryPublishResult, Failure> > TryStoreFingerprintEntryAsync(
ContentFingerprint fingerprint,
PipFingerprintEntry entry,
PipFingerprintEntry previousEntry = null,
bool replaceExisting = true,
CacheQueryData cacheQueryData = null)
{
Contract.Assume(entry != null);
Analysis.IgnoreArgument(previousEntry); // See class remarks; replace semantics are broken.
// We have in hand a PipFingerprintEntry which the underlyign m_twoPhaseStore does not understand.
// We will serialize it and store it to the CAS, and that CAS hash will be the stored entry's MetadataHash.
// See symmetric deserialization in TryGetFingerprintEntryAsync.
Possible <ContentHash, Failure> maybeStored = await m_contentCache.TrySerializeAndStoreContent(entry);
if (!maybeStored.Succeeded)
{
return(maybeStored.Failure.Annotate("Failed to store cache-entry metadata to the CAS"));
}
ContentHash metadataHash = maybeStored.Result;
// The metadata (single-phase entry) is stored, so now we can construct an entry that references it.
// From now on, 'twoPhaseEntry' will mean 'the entry we are actually storing in the two-phase store'.
// Meanwhile, like any implementation, we assume that the referenced content (e.g. output files)
// were stored by the caller already.
ContentHash[] twoPhaseReferencedHashes = entry.OutputContentHashes.Select(b => b.ToContentHash()).Where(hash => !hash.IsSpecialValue()).ToArray();
CacheEntry twoPhaseEntry = new CacheEntry(metadataHash, null, ArrayView <ContentHash> .FromArray(twoPhaseReferencedHashes));
StrongContentFingerprint dummyFingerprint = ComputeDummyStrongFingerprint(
m_pathTable,
fingerprint);
var weakFingerprint = new WeakContentFingerprint(fingerprint.Hash);
Possible <CacheEntryPublishResult, Failure> maybePublished = await m_twoPhaseStore.TryPublishCacheEntryAsync(
weakFingerprint : weakFingerprint,
pathSetHash : s_dummyPathSetHash,
strongFingerprint : dummyFingerprint,
entry : twoPhaseEntry,
mode : replaceExisting?CacheEntryPublishMode.CreateNewOrReplaceExisting : CacheEntryPublishMode.CreateNew);
if (cacheQueryData != null)
{
cacheQueryData.WeakContentFingerprint = weakFingerprint;
cacheQueryData.PathSetHash = s_dummyPathSetHash;
cacheQueryData.StrongContentFingerprint = dummyFingerprint;
cacheQueryData.ContentCache = m_contentCache;
}
if (maybePublished.Succeeded)
{
if (maybePublished.Result.Status == CacheEntryPublishStatus.Published ||
(!replaceExisting && maybePublished.Result.Status == CacheEntryPublishStatus.RejectedDueToConflictingEntry))
{
return(maybePublished.Result);
}
else
{
// ISinglePhaseFingerprintStore represents conflicts as failures.
return(new Failure <string>(
"Failed to publish a cache entry; the underlying two-phase store indicated an entry conflict (maybe it does not allow replacement of existing entries)."));
}
}
else
{
return(maybePublished.Failure);
}
}
/// <summary>
/// Writes fingerprints.
/// </summary>
internal static void Write(this BinaryWriter writer, ContentFingerprint fingerprint, byte[] buffer)
{
Array.Clear(buffer, 0, buffer.Length);
fingerprint.Hash.Serialize(buffer);
writer.Write(buffer);
}
/// <summary>
/// Creates an instance of <see cref="PipWithFingerprint"/>.
/// </summary>
public PipWithFingerprint(Pip pip, ContentFingerprint fingerprint, string fingerprintText)
{
Pip = pip;
Fingerprint = fingerprint;
FingerprintText = fingerprintText;
}
