/// <inheritdoc />
public IEnumerable <Task <Possible <StrongFingerprint, Failure> > > EnumerateStrongFingerprints(
WeakFingerprintHash weak, UrgencyHint urgencyHint, Guid activityId)
{
// TODO: Extend IAsyncEnumerable up through EnumerateStrongFingerprints
var tcs = TaskSourceSlim.Create <IEnumerable <GetSelectorResult> >();
yield return(Task.Run(
async() =>
{
try
{
var results = await ReadOnlyCacheSession.GetSelectors(new Context(Logger), weak.ToMemoization(), CancellationToken.None).ToListAsync();
tcs.SetResult(results);
return results.Any() ? results.First().FromMemoization(weak, CacheId) : StrongFingerprintSentinel.Instance;
}
catch (Exception ex)
{
tcs.SetException(ex);
throw;
}
}));
// For now, callers should always await the first task before enumerating the rest
Contract.Assert(tcs.Task.IsCompleted);
IEnumerable <GetSelectorResult> otherResults = tcs.Task.GetAwaiter().GetResult();
foreach (var otherResult in otherResults.Skip(1))
{
yield return(Task.FromResult(otherResult.FromMemoization(weak, CacheId)));
}
}
/// <summary>
/// Failure of an observed input list filter
/// </summary>
/// <param name="cacheId">Cache ID where failure happened</param>
/// <param name="weak">Weak fingerprint component of the strong fingerprint</param>
/// <param name="casElement">The CasElement component of the strong fingerprint</param>
/// <param name="hashElement">The HashElement component of the strong fingerprint</param>
/// <param name="message">Details about the failure</param>
public InputListFilterFailure(string cacheId, WeakFingerprintHash weak, CasHash casElement, Hash hashElement, string message)
{
Contract.Requires(message != null);
m_strongFingerprint = new StrongFingerprint(weak, casElement, hashElement, cacheId);
m_message = message;
}
/// <summary>
/// Creates a FakeStrongFingerpint and the associated FakeBuild streams.
/// </summary>
/// <param name="pipName">Name of the pip</param>
/// <param name="generateVerifiablePip">Determines if CheckContentsAsync can verify the contents of the build</param>
/// <param name="pipSize">Number of files to include in the pip</param>
/// <remarks>
/// This method is the StrongFingerprint generation algorithm for FakeBuild.DoNondeterministicPipAsync
/// </remarks>
/// <returns>A FakeStrongFingerprint that has the FakeBuild class</returns>
public static FakeStrongFingerprint Create(string pipName, bool generateVerifiablePip = false, int pipSize = 3)
{
FakeBuild fake = new FakeBuild(pipName, pipSize, forceUniqueOutputs: !generateVerifiablePip);
WeakFingerprintHash weak = CreateWeak(pipName.ToLowerInvariant());
Hash simpleHash = CreateHash(pipName.ToUpperInvariant());
return(new FakeStrongFingerprint(weak, simpleHash, fake));
}
private async Task <Possible <FullCacheRecordWithDeterminism, Failure> > AddToCache(ICacheSession session, params string[] thePaths)
{
WeakFingerprintHash weak = FakeStrongFingerprint.CreateWeak(session.CacheId);
CasHash casHash = await AddPathSet(session, thePaths);
Hash simpleHash = FakeStrongFingerprint.CreateHash(session.CacheSessionId);
return(await session.AddOrGetAsync(weak, casHash, simpleHash, CasEntries.FromCasHashes(casHash)));
}
/// <summary>
/// Create the failure, including the weak fingerprint for which enumeration failed
/// </summary>
/// <param name="cacheId">The cacheId where the failure happened</param>
/// <param name="weak">The weak fingerprint for which enumeration was attempted</param>
/// <param name="rootCause">Optional root cause exception</param>
public StrongFingerprintEnumerationFailure(string cacheId, WeakFingerprintHash weak, Exception rootCause)
{
Contract.Requires(cacheId != null);
Contract.Requires(rootCause != null);
m_cacheId = cacheId;
m_weak = weak;
m_rootCause = rootCause;
}
public void SelectorResultSuccessFromMemoization()
{
WeakFingerprintHash weak = RandomHelpers.CreateRandomWeakFingerprintHash();
GetSelectorResult selectorResult = new GetSelectorResult(Selector.Random(HashType.Vso0, FingerprintUtilities.FingerprintLength));
Possible <BuildXLStrongFingerprint, Failure> maybeStrongFingerprint = selectorResult.FromMemoization(weak, CacheId);
Assert.True(maybeStrongFingerprint.Succeeded);
Assert.Equal(weak, maybeStrongFingerprint.Result.WeakFingerprint);
Assert.Equal(selectorResult.Selector.ContentHash.FromMemoization(), maybeStrongFingerprint.Result.CasElement);
Assert.Equal(selectorResult.Selector.Output, maybeStrongFingerprint.Result.HashElement.ToArray());
Assert.Equal(CacheId, maybeStrongFingerprint.Result.CacheId);
}
/// <summary>
/// Iterates over input strong fingerprints and returns all unique weak fingerprints
/// </summary>
/// <param name="strongFingerprints">Strong fingerprints to iterate over</param>
/// <returns>All unique weak fingerprints found among input strong fingerprints</returns>
public static IEnumerable <WeakFingerprintHash> ProduceWeakFingerprints(this IEnumerable <StrongFingerprint> strongFingerprints)
{
HashSet <WeakFingerprintHash> weakFingerprintsAlreadyFound = new HashSet <WeakFingerprintHash>();
foreach (StrongFingerprint strongFingerprint in strongFingerprints)
{
WeakFingerprintHash weakFingerprint = strongFingerprint.WeakFingerprint;
if (weakFingerprintsAlreadyFound.Add(weakFingerprint))
{
yield return(weakFingerprint);
}
}
}
/// <summary>
/// Writes the Start Activity event
/// </summary>
public void Start(WeakFingerprintHash weak, UrgencyHint urgencyHint)
{
Start();
if (TraceMethodArgumentsEnabled())
{
Write(
ParameterOptions,
new
{
WeakFingerprintHash = weak,
UrgencyHint = urgencyHint,
});
}
}
/// <summary>
/// Writes the Start Activity event
/// </summary>
public void Start(WeakFingerprintHash weak, CasHash casElement, Hash hashElement, CasEntries hashes, UrgencyHint urgencyHint)
{
Start();
if (TraceMethodArgumentsEnabled())
{
Write(
ParameterOptions,
new
{
WeakFingerprintHash = weak,
CasHash = casElement,
Hash = hashElement,
CasEntries = hashes,
UrgencyHint = urgencyHint,
});
}
}
public async Task AddWithoutPinWeak()
{
const string TestName = nameof(AddWithoutPinWeak);
string testCacheId = MakeCacheId(TestName);
ICache cache = await CreateCacheAsync(testCacheId, strictMetadataCasCoupling : false);
// Only caches that are not strict metadata to CAS coupling need this test
if (cache.StrictMetadataCasCoupling)
{
(await cache.ShutdownAsync()).Success();
return;
}
string testSessionId = "Session1-" + testCacheId;
ICacheSession session = await CreateSessionAsync(cache, testSessionId);
FakeBuild fake = new FakeBuild(TestName, 2);
// We fake up a CasHash - never actually add to the cache - weak cas
CasHash inputList = fake.OutputListHash;
CasHash[] items = new CasHash[fake.Outputs.Length];
for (int i = 0; i < fake.Outputs.Length; i++)
{
items[i] = new CasHash(fake.OutputHashes[i]);
}
// We use the hash of our output list as the weak fingerprint and extra hash
var outputListFingerprintHash = fake.OutputListHash.ToFingerprint();
WeakFingerprintHash weak = new WeakFingerprintHash(outputListFingerprintHash);
// This should work since the session is weak.
FullCacheRecordWithDeterminism record = (await session.AddOrGetAsync(weak, inputList, outputListFingerprintHash, items)).Success();
XAssert.IsNull(record.Record, "There should not have been anything in the cache");
// Doing it twice does not change things...
record = (await session.AddOrGetAsync(weak, inputList, outputListFingerprintHash, items)).Success();
XAssert.IsNull(record.Record, "It matches exactly, so no bother");
await CloseSessionAsync(session, testSessionId);
await ShutdownCacheAsync(cache, testCacheId);
}
/// <inheritdoc />
public IEnumerable <Task <Possible <PublishedEntryRef, Failure> > > ListPublishedEntriesByWeakFingerprint(WeakContentFingerprint weak)
{
WeakFingerprintHash cacheCoreWeakFingerprint = new WeakFingerprintHash(new Hash(weak.Hash));
// TODO: We assume that everything up until the first sentinel is local. This is fine for a simple 'vertical' aggregator
// of a local and remote cache, but isn't general.
PublishedEntryRefLocality currentLocality = PublishedEntryRefLocality.Local;
foreach (Task <Possible <StrongFingerprint, Failure> > entryPromise in m_cache.EnumerateStrongFingerprints(cacheCoreWeakFingerprint))
{
if (entryPromise.IsCompleted && entryPromise.Result.Succeeded && entryPromise.Result.Result is StrongFingerprintSentinel)
{
currentLocality = PublishedEntryRefLocality.Remote;
continue;
}
yield return(AdaptPublishedEntry(entryPromise, currentLocality));
}
}
public async Task CacheMissTest()
{
const string TestName = nameof(CacheMissTest);
string testCacheId = MakeCacheId(TestName);
ICache cache = await CreateCacheAsync(testCacheId);
ICacheSession session = await cache.CreateSessionAsync().SuccessAsync();
FakeBuild fb = new FakeBuild("test", 3);
StrongFingerprint fakeFingerprint = new StrongFingerprint(WeakFingerprintHash.Random(), new CasHash(fb.OutputHashes[0]), fb.OutputHashes[0], "fake");
var response = await session.GetCacheEntryAsync(fakeFingerprint);
XAssert.IsFalse(response.Succeeded);
XAssert.AreEqual(typeof(NoMatchingFingerprintFailure), response.Failure.GetType(), response.Failure.Describe());
await CloseSessionAsync(session, null);
await ShutdownCacheAsync(cache, testCacheId);
}
private static async Task <FullCacheRecord> DoPipAsyncImpl(ICacheSession session,
WeakFingerprintHash weak,
Hash simpleHash,
FakeBuild fake,
CacheDeterminism determinism,
CasAccessMethod accessMethod)
{
foreach (var strongTask in session.EnumerateStrongFingerprints(weak))
{
StrongFingerprint possibleHit = await strongTask.SuccessAsync();
if (fake.OutputListHash.Equals(possibleHit.CasElement))
{
if (simpleHash.Equals(possibleHit.HashElement))
{
// A cache hit! Our strong fingerprint matched
return(new FullCacheRecord(possibleHit, await session.GetCacheEntryAsync(possibleHit).SuccessAsync()));
}
}
}
// A cache miss - add the content to the cache and then
// add the build.
CasHash inputList = await AddToCasAsync(fake.OutputList, accessMethod, session).SuccessAsync();
CasHash[] items = new CasHash[fake.Outputs.Length];
for (int i = 0; i < items.Length; i++)
{
items[i] = await AddToCasAsync(fake.Outputs[i], accessMethod, session).SuccessAsync();
}
CasEntries entries = new CasEntries(items, determinism);
FullCacheRecordWithDeterminism cacheRecord = await session.AddOrGetAsync(weak, inputList, simpleHash, entries).SuccessAsync();
XAssert.AreEqual(null, cacheRecord.Record);
// Produce a full cache record manually - such that the CacheId is our own "NewRecordCacheId"
return(new FullCacheRecord(new StrongFingerprint(weak, inputList, simpleHash, NewRecordCacheId), entries));
}
/// <summary>
/// This will do a build into the session with a given name
/// and output count.
/// </summary>
/// <param name="session">The cache session to work with</param>
/// <param name="pipName">Some text that acts as a base element in the output</param>
/// <param name="pipSize">Number of elements in the output. Must be enough to cover the variants</param>
/// <param name="weakIndex">Variant with different weak index - defaults to 1</param>
/// <param name="hashIndex">Variant with different hash index - defaults to 0</param>
/// <param name="accessMethod">Method (File or stream) for how files are materialized from the cache</param>
/// <param name="determinism">Determinism to provide for new build records</param>
/// <returns>The FullCacheRecord of the build</returns>
/// <remarks>
/// This will do a "fake build" including a cache lookup via weak fingerprints
/// and then return either the existing FullCacheRecord or add the build as a new
/// one. A new FullCacheRecord will have the StrongFingerprint.CacheId set to NewRecordCacheId
/// </remarks>
public static Task <FullCacheRecord> DoPipAsync(
ICacheSession session,
string pipName,
int pipSize = DefaultFakeBuildSize,
int weakIndex = 1,
int hashIndex = 0,
CacheDeterminism determinism = default(CacheDeterminism),
CasAccessMethod accessMethod = CasAccessMethod.Stream)
{
Contract.Requires(session != null);
Contract.Requires(pipName != null);
Contract.Requires(pipSize > 0);
Contract.Requires(weakIndex >= 0 && weakIndex < pipSize);
Contract.Requires(hashIndex >= 0 && hashIndex < pipSize);
FakeBuild fake = new FakeBuild(pipName, pipSize);
WeakFingerprintHash weak = new WeakFingerprintHash(FingerprintUtilities.Hash(fake.OutputHashes[weakIndex].ToString()).ToByteArray());
Hash simpleHash = new Hash(FingerprintUtilities.Hash(fake.OutputHashes[hashIndex].ToString()));
return(DoPipAsyncImpl(session, weak, simpleHash, fake, determinism, accessMethod));
}
public Task <Possible <FullCacheRecordWithDeterminism, Failure> > AddOrGetAsync(WeakFingerprintHash weak, CasHash casElement, Hash hashElement, CasEntries hashes, UrgencyHint urgencyHint, Guid activityId)
{
var callback = AddOrGetAsyncCallback;
if (callback != null)
{
return(callback(weak, casElement, hashElement, hashes, urgencyHint, activityId, m_realSession));
}
else
{
return(m_realSession.AddOrGetAsync(weak, casElement, hashElement, hashes, urgencyHint, activityId));
}
}
public IEnumerable <Task <Possible <StrongFingerprint, Failure> > > EnumerateStrongFingerprints(WeakFingerprintHash weak, UrgencyHint urgencyHint, Guid activityId)
{
return(m_session.EnumerateStrongFingerprints(weak, urgencyHint, activityId));
}
/// <summary>
/// Create a random WeakFingerprintHash
/// </summary>
public static WeakFingerprintHash CreateRandomWeakFingerprintHash()
{
return(WeakFingerprintHash.Random());
}
private Possible <FullCacheRecordWithDeterminism, Failure> AddOrGet(WeakFingerprintHash weak, CasHash casElement, BuildXL.Cache.Interfaces.Hash hashElement, CasEntries hashes)
{
Contract.Requires(!IsClosed);
Contract.Requires(hashes.IsValid);
Contract.Assert(!m_readOnly);
// We check the Cas entries if we are strict
if (StrictMetadataCasCoupling)
{
// Check that the content is valid.
if (!m_pinnedToCas.ContainsKey(casElement))
{
return(new UnpinnedCasEntryFailure(CacheId, casElement));
}
foreach (CasHash hash in hashes)
{
if (!m_pinnedToCas.ContainsKey(hash))
{
return(new UnpinnedCasEntryFailure(CacheId, hash));
}
}
}
var strongFingerprints = Cache.Fingerprints.GetOrAdd(weak, (key) => new ConcurrentDictionary <StrongFingerprint, FullCacheRecord>());
var record = strongFingerprints.AddOrUpdate(
new StrongFingerprint(weak, casElement, hashElement, Cache.CacheId),
(strong) => new FullCacheRecord(strong, hashes),
(strong, oldRecord) =>
{
// Do no harm here - we will recheck this outside to produce the error that it was a bad attempt
if (oldRecord.CasEntries.Determinism.IsSinglePhaseNonDeterministic != hashes.Determinism.IsSinglePhaseNonDeterministic)
{
return(oldRecord);
}
// We replace if we are SinglePhaseNonDeterministic *or*
// if we are upgrading the determinism.
if (hashes.Determinism.IsSinglePhaseNonDeterministic ||
(!oldRecord.CasEntries.Determinism.IsDeterministicTool &&
(hashes.Determinism.IsDeterministic && !hashes.Determinism.Equals(oldRecord.CasEntries.Determinism))) ||
HasMissingContent(oldRecord.CasEntries))
{
oldRecord = new FullCacheRecord(strong, hashes);
}
return(oldRecord);
});
if (record.CasEntries.Determinism.IsSinglePhaseNonDeterministic != hashes.Determinism.IsSinglePhaseNonDeterministic)
{
return(new SinglePhaseMixingFailure(CacheId));
}
AddSessionRecord(record);
if (record.CasEntries.Equals(hashes))
{
record = null;
}
else
{
// Check if tool determinism did not make it in - that means a very bad thing happened
if (hashes.Determinism.IsDeterministicTool)
{
return(new NotDeterministicFailure(Cache.CacheId, record, new FullCacheRecord(record.StrongFingerprint, hashes)));
}
}
if (record == null)
{
return(new FullCacheRecordWithDeterminism(hashes.GetFinalDeterminism(Cache.IsAuthoritative, Cache.CacheGuid, CacheDeterminism.NeverExpires)));
}
else
{
return(new FullCacheRecordWithDeterminism(new FullCacheRecord(record.StrongFingerprint, record.CasEntries.GetModifiedCasEntriesWithDeterminism(Cache.IsAuthoritative, Cache.CacheGuid, CacheDeterminism.NeverExpires))));
}
}
public async Task AddWithoutPin()
{
const string TestName = nameof(AddWithoutPin);
string testCacheId = MakeCacheId(TestName);
ICache cache = await CreateCacheAsync(testCacheId);
string testSessionId = "Session1-" + testCacheId;
ICacheSession session = await CreateSessionAsync(cache, testSessionId);
FakeBuild fake = new FakeBuild(TestName, 2);
CasHash inputList = (await session.AddToCasAsync(fake.OutputList)).Success();
// Verify that we can read the content after it was added in
// this session since it was pinned
using (Stream stream = (await session.GetStreamAsync(inputList)).Success())
{
stream.AsString();
}
CasHash[] items = new CasHash[fake.Outputs.Length];
for (int i = 0; i < fake.Outputs.Length; i++)
{
items[i] = (await session.AddToCasAsync(fake.Outputs[i])).Success();
// Verify that we can read the content after it was added in
// this session since it was pinned
using (Stream stream = (await session.GetStreamAsync(items[i])).Success())
{
stream.AsString();
}
}
await CloseSessionAsync(session, testSessionId);
testSessionId = "Session2-" + testCacheId;
session = await CreateSessionAsync(cache, testSessionId);
// We use the hash of our output list as the weak fingerprint and extra hash
var outputListFingerprintHash = fake.OutputListHash.ToFingerprint();
WeakFingerprintHash weak = new WeakFingerprintHash(outputListFingerprintHash);
if (cache.StrictMetadataCasCoupling)
{
// This should fail as we did not pin or add the content yet
var error = await session.AddOrGetAsync(weak, inputList, outputListFingerprintHash, items);
XAssert.IsFalse(error.Succeeded, "Add of weak fingerprint when items not first added to CAS was successful.");
XAssert.IsTrue(error.Failure is UnpinnedCasEntryFailure, "Error failure was not expected type {0}, it was {1} {2}", typeof(UnpinnedCasEntryFailure).Name, error.Failure.GetType().Name, error.Failure.Describe());
// Doing it twice does not change things...
error = await session.AddOrGetAsync(weak, inputList, outputListFingerprintHash, items);
XAssert.IsFalse(error.Succeeded, "Add of weak fingerprint when items not first added to CAS was successful.");
XAssert.IsTrue(error.Failure is UnpinnedCasEntryFailure, "Error failure was not expected type {0}, it was {1}, {2}", typeof(UnpinnedCasEntryFailure).Name, error.Failure.GetType().Name, error.Failure.Describe());
}
await CloseSessionAsync(session, testSessionId);
await ShutdownCacheAsync(cache, testCacheId);
}
public IEnumerable <Task <Possible <StrongFingerprint, Failure> > > EnumerateStrongFingerprints(WeakFingerprintHash weak, UrgencyHint urgencyHint, Guid activityId)
{
ConcurrentDictionary <StrongFingerprint, FullCacheRecord> strongFingerprints;
if (Cache.Fingerprints.TryGetValue(weak, out strongFingerprints))
{
foreach (StrongFingerprint strongFingerprint in strongFingerprints.Keys)
{
yield return(Task.FromResult(new Possible <StrongFingerprint, Failure>(strongFingerprint)));
}
}
}
public Task <Possible <FullCacheRecordWithDeterminism, Failure> > AddOrGetAsync(WeakFingerprintHash weak, CasHash casElement, BuildXL.Cache.Interfaces.Hash hashElement, CasEntries hashes, UrgencyHint urgencyHint, Guid activityId)
{
Contract.Requires(!IsClosed);
Contract.Requires(hashes.IsValid);
Contract.Assert(!m_readOnly);
return(Task.Run(() => AddOrGet(weak, casElement, hashElement, hashes)));
}
public async Task <Possible <FullCacheRecordWithDeterminism, Failure> > AddOrGetAsync(WeakFingerprintHash weak, CasHash casElement, Hash hashElement, CasEntries hashes, UrgencyHint urgencyHint, Guid activityId)
{
var check = await CheckInputList(weak, casElement, hashElement, urgencyHint, activityId);
if (!check.Succeeded)
{
return(check.Failure);
}
return(await m_session.AddOrGetAsync(weak, casElement, hashElement, hashes, urgencyHint, activityId));
}
/// <summary>
/// Check the input list against the regex
/// </summary>
/// <param name="weak">The weak fingerprint (for logging on failure)</param>
/// <param name="casElement">The CasElement of the strong fingerprint</param>
/// <param name="hashElement">The hashElement of the strong fingerprint (for logging on failure)</param>
/// <param name="urgencyHint">Pass-through</param>
/// <param name="activityId">Pass-through activityId</param>
/// <returns>false if the check was not performed, true if the checks were performed, failure if the regex checks failed</returns>
/// <remarks>
/// This will attempt to validate the CAS stored input list against the regex rules
/// </remarks>
private async Task <Possible <bool, Failure> > CheckInputList(WeakFingerprintHash weak, CasHash casElement, Hash hashElement, UrgencyHint urgencyHint, Guid activityId)
{
// If we either have no CasHash item or we have no regex to check, just return false
// (that we did nothing)
if (casElement.Equals(CasHash.NoItem) || ((Cache.MustIncludeRegex == null) && (Cache.MustNotIncludeRegex == null)))
{
return(false);
}
// mustInclude start out false if we need to check for mustInclude
// Once we get a mustInclude match we no longer need to check.
// If we have no mustInclude regex, we set it to true such that
// we don't bother checking it
bool mustInclude = Cache.MustIncludeRegex == null;
// This is just to make a faster check for the MustNotinclude
// case. If we have the regex then we must check each entry
// but in many cases we don't have the regex so let this be a quick out.
bool checkMustNot = Cache.MustNotIncludeRegex != null;
// Try to get the observed inputs from the CasHash given
var possibleStream = await GetStreamAsync(casElement, urgencyHint, activityId);
if (!possibleStream.Succeeded)
{
// If we could not get a stream to the CasEntery in the fingerprint.
return(new InputListFilterFailure(Cache.CacheId, weak, casElement, hashElement, "Failed to get stream of CasElement"));
}
// Deserialize the contents of the path set.
using (possibleStream.Result)
{
PathTable pathTable = new PathTable();
BuildXLReader reader = new BuildXLReader(false, possibleStream.Result, true);
var maybePathSet = ObservedPathSet.TryDeserialize(pathTable, reader);
if (maybePathSet.Succeeded)
{
// Deserialization was successful
foreach (ObservedPathEntry entry in maybePathSet.Result.Paths)
{
string filepath = entry.Path.ToString(pathTable);
// Have we seen a must-have entry yet? If not check if this is one
// that way once we found one we want we stop checking this regex
if (!mustInclude)
{
mustInclude = Cache.MustIncludeRegex.IsMatch(filepath);
}
// Now, if we are looking for a must not include, we just check for that
// and if it matches we fail
if (checkMustNot)
{
if (Cache.MustNotIncludeRegex.IsMatch(filepath))
{
return(new InputListFilterFailure(Cache.CacheId, weak, casElement, hashElement, string.Format(CultureInfo.InvariantCulture, "Failed due to a MustNotInclude file: {0}", filepath)));
}
}
}
}
else
{
return(new InputListFilterFailure(Cache.CacheId, weak, casElement, hashElement, "Failed to deserialize observed inputs"));
}
}
if (!mustInclude)
{
return(new InputListFilterFailure(Cache.CacheId, weak, casElement, hashElement, "Failed due to not including at least one MustInclude file"));
}
return(true);
}
public IEnumerable <Task <Possible <StrongFingerprint, Failure> > > EnumerateStrongFingerprints(WeakFingerprintHash weak, UrgencyHint urgencyHint, Guid activityId)
{
using (var eventing = new EnumerateStrongFingerprintsActivity(CompositingCache.EventSource, activityId, this))
{
eventing.Start(weak, urgencyHint);
var ret = m_metadataSession.EnumerateStrongFingerprints(weak, urgencyHint, eventing.Id);
eventing.Stop();
return(ret);
}
}
public async Task <Possible <FullCacheRecordWithDeterminism, Failure> > AddOrGetAsync(WeakFingerprintHash weak, CasHash casElement, Hash hashElement, CasEntries hashes, UrgencyHint urgencyHint, Guid activityId)
{
var addResult = await CacheSession.AddOrGetContentHashListAsync(
new Context(Logger),
new BuildXL.Cache.MemoizationStore.Interfaces.Sessions.StrongFingerprint(
weak.ToMemoization(),
new Selector(casElement.ToMemoization(), hashElement.RawHash.ToByteArray())),
hashes.ToMemoization(),
CancellationToken.None);
var strong = new StrongFingerprint(weak, casElement, hashElement, CacheId);
switch (addResult.Code)
{
case AddOrGetContentHashListResult.ResultCode.Success:
SessionEntries?.TryAdd(strong, 1);
return(addResult.ContentHashListWithDeterminism.ContentHashList == null
? new FullCacheRecordWithDeterminism(addResult.ContentHashListWithDeterminism.Determinism.FromMemoization())
: new FullCacheRecordWithDeterminism(new FullCacheRecord(strong, addResult.ContentHashListWithDeterminism.FromMemoization())));
case AddOrGetContentHashListResult.ResultCode.SinglePhaseMixingError:
return(new SinglePhaseMixingFailure(CacheId));
case AddOrGetContentHashListResult.ResultCode.InvalidToolDeterminismError:
return(new NotDeterministicFailure(
CacheId,
new FullCacheRecord(strong, addResult.ContentHashListWithDeterminism.FromMemoization()),
new FullCacheRecord(strong, hashes)));
case AddOrGetContentHashListResult.ResultCode.Error:
return(new CacheFailure(addResult.ErrorMessage));
default:
return(new CacheFailure("Unrecognized AddOrGetContentHashListAsync result code: " + addResult.Code + ", error message: " + (addResult.ErrorMessage ?? string.Empty)));
}
}
public IEnumerable <Task <Possible <StrongFingerprint, Failure> > > EnumerateStrongFingerprints(WeakFingerprintHash weak, UrgencyHint urgencyHint, Guid activityId)
{
var callback = EnumerateStrongFingerprintsCallback;
if (callback != null)
{
return(callback(weak, urgencyHint, activityId, m_realSession));
}
else
{
return(m_realSession.EnumerateStrongFingerprints(weak, urgencyHint, activityId));
}
}
public async Task <Possible <FullCacheRecordWithDeterminism, Failure> > AddOrGetAsync(WeakFingerprintHash weak, CasHash casElement, Hash hashElement, CasEntries hashes, UrgencyHint urgencyHint, Guid activityId)
{
using (var eventing = new AddOrGetActivity(CompositingCache.EventSource, activityId, this))
{
eventing.Start(weak, casElement, hashElement, hashes, urgencyHint);
// First, check if all the content has been pinned.
if (Cache.StrictMetadataCasCoupling)
{
List <CasHash> referencedHashes = new List <CasHash>(hashes);
referencedHashes.Add(casElement);
foreach (CasHash oneHash in referencedHashes)
{
if (!PinnedToCas.ContainsKey(oneHash))
{
return(eventing.StopFailure(new UnpinnedCasEntryFailure(Cache.CacheId, oneHash)));
}
}
}
// Then check the determinism bit...
var testStrongFingerprint = new StrongFingerprint(weak, casElement, hashElement, Cache.CacheId);
var existRecordCheck = await m_metadataSession.GetCacheEntryAsync(testStrongFingerprint, urgencyHint, eventing.Id);
if (existRecordCheck.Succeeded)
{
// There's an existing record, so we need the rules for determinism upgrade / downgrade. Which are complicated by the existance
// (or lack there of) of the associated content.
// First we'll check for determinism upgrades or other situations that don't require content probes.
var existingRecord = existRecordCheck.Result;
// If the determinsim is going from no determinism to some determinism OR the record is going from a non-tool
// determinism to a tool determinism, replace
if ((existingRecord.Determinism.IsNone && hashes.Determinism.IsDeterministic) ||
(hashes.Determinism.IsDeterministicTool && !existingRecord.Determinism.IsDeterministicTool))
{
return(eventing.Returns(await m_metadataSession.AddOrGetAsync(weak, casElement, hashElement, hashes, urgencyHint, eventing.Id)));
}
// Before trying to probe for files, see if the entries are identical.
// If so, tell the caller we stored their record.
if (hashes == existingRecord && hashes.Determinism.EffectiveGuid == existingRecord.Determinism.EffectiveGuid)
{
return(eventing.Returns(new FullCacheRecordWithDeterminism(CacheDeterminism.None)));
}
// The rest of the determinism moves depend on if the files exist or not.
bool foundAllFiles = true;
var pinCheck = await m_casSession.PinToCasAsync(existRecordCheck.Result, urgencyHint, eventing.Id);
foreach (var oneOutput in pinCheck)
{
if (!oneOutput.Succeeded)
{
foundAllFiles = false;
}
}
// Ok, so now with the knowledge of file existance, if we're missing files, allow the call through no matter what.
if (!foundAllFiles)
{
return(eventing.Returns(await m_metadataSession.AddOrGetAsync(weak, casElement, hashElement, hashes, urgencyHint, eventing.Id)));
}
// If the existing entry is more deterministic than what we're being given or
// the records are the same level, unless tool deterministic or single phase.
if ((existingRecord.Determinism.IsDeterministic && !hashes.Determinism.IsDeterministic) ||
(existingRecord.Determinism.IsDeterministicTool && !hashes.Determinism.IsDeterministicTool) ||
(existingRecord.Determinism.EffectiveGuid == hashes.Determinism.EffectiveGuid && !existingRecord.Determinism.IsDeterministicTool && !existingRecord.Determinism.IsSinglePhaseNonDeterministic))
{
// If the records are identical except for determinsim, return null.
if (existingRecord == hashes)
{
return(eventing.Returns(new FullCacheRecordWithDeterminism(CacheDeterminism.None)));
}
return(eventing.Returns(new FullCacheRecordWithDeterminism(new FullCacheRecord(testStrongFingerprint, existingRecord))));
}
// And now the error conditions.
// If a tool determinism collection, or an attempt to go from deterministic to not deterministic.
if (existingRecord.Determinism.IsDeterministicTool && hashes.Determinism.IsDeterministicTool)
{
return(eventing.StopFailure(new NotDeterministicFailure(Cache.CacheId, new FullCacheRecord(testStrongFingerprint, existingRecord), new FullCacheRecord(testStrongFingerprint, hashes))));
}
}
return(eventing.Returns(await m_metadataSession.AddOrGetAsync(weak, casElement, hashElement, hashes, urgencyHint, eventing.Id)));
}
}
public IEnumerable <Task <Possible <StrongFingerprint, Failure> > > EnumerateStrongFingerprints(WeakFingerprintHash weak, UrgencyHint urgencyHint, Guid activityId)
{
Contract.Requires(!IsClosed);
using (var counter = m_counters.EnumerateStrongFingerprintsCounter())
{
using (var eventing = new EnumerateStrongFingerprintsActivity(BasicFilesystemCache.EventSource, activityId, this))
{
eventing.Start(weak, urgencyHint);
// It's possible the cache could encounter an IO error when attempting to enumerate the fingerprints.
// This shouldn't be a fatal error. we just want to catch and record it, then continue on.
// Due to the use of yield a foreach loop can't be used here while still handling the error,
// so the enumation must be done manually.
using (var enumerator = m_cache.EnumerateStrongFingerprints(weak).GetEnumerator())
{
while (true)
{
StrongFingerprint strong = null;
try
{
if (!enumerator.MoveNext())
{
break;
}
strong = enumerator.Current;
}
catch (IOException ex)
{
eventing.StopFailure(new StrongFingerprintEnumerationFailure(m_cache.CacheId, weak, ex));
yield break;
}
catch (UnauthorizedAccessException ex)
{
eventing.StopFailure(new StrongFingerprintEnumerationFailure(m_cache.CacheId, weak, ex));
yield break;
}
counter.YieldReturn();
yield return(Task.FromResult(new Possible <StrongFingerprint, Failure>(strong)));
}
}
eventing.Stop();
}
}
}
/// <summary>
/// Responsible for parsing the supplied command
/// line arguments
/// </summary>
/// <param name="args">
/// Array of the command line arguments
/// </param>
public Args(string[] args)
: base(args)
{
string jsonFilePath = null;
string sessionIDFilter = null;
string inputAssertionListDumpMustIncludeRegex = null;
string inputAssertionListDumpMustNotIncludeRegex = null;
string outputFileName = null;
string weakFingerprintInputFilepath = null;
foreach (Option opt in Options)
{
if (s_helpStrings.Any(s => opt.Name.Equals(s, StringComparison.OrdinalIgnoreCase)))
{
Help = true;
WriteHelp();
return;
}
else if (opt.Name.Equals("statisticalAnalysis", StringComparison.OrdinalIgnoreCase) ||
opt.Name.Equals("sa", StringComparison.OrdinalIgnoreCase))
{
m_runStatisticalAnalysis = true;
}
else if (opt.Name.Equals("contentAnalysis", StringComparison.OrdinalIgnoreCase) ||
opt.Name.Equals("ca", StringComparison.OrdinalIgnoreCase))
{
m_analyzeContent = true;
}
else if (opt.Name.Equals("consistencyCheck", StringComparison.OrdinalIgnoreCase) ||
opt.Name.Equals("cc", StringComparison.OrdinalIgnoreCase))
{
m_runConsistencyCheck = true;
}
else if (opt.Name.Equals("inputAssertionListCheck", StringComparison.OrdinalIgnoreCase) ||
opt.Name.Equals("ic", StringComparison.OrdinalIgnoreCase))
{
m_findInputAssertionListAnomalies = true;
}
else if (opt.Name.Equals("contentBreakdown", StringComparison.OrdinalIgnoreCase) ||
opt.Name.Equals("cb", StringComparison.OrdinalIgnoreCase))
{
m_runContentBreakdown = true;
}
else if (opt.Name.Equals("jsonString", StringComparison.OrdinalIgnoreCase) ||
opt.Name.Equals("js", StringComparison.OrdinalIgnoreCase))
{
m_jsonString = opt.Value;
}
else if (opt.Name.Equals("jsonFile", StringComparison.OrdinalIgnoreCase) ||
opt.Name.Equals("jf", StringComparison.OrdinalIgnoreCase))
{
jsonFilePath = opt.Value;
}
else if (opt.Name.Equals("sessionIDFilter", StringComparison.OrdinalIgnoreCase) ||
opt.Name.Equals("sf", StringComparison.OrdinalIgnoreCase))
{
sessionIDFilter = opt.Value;
}
else if (opt.Name.Equals("rehashCASContent", StringComparison.OrdinalIgnoreCase) ||
opt.Name.Equals("rc", StringComparison.OrdinalIgnoreCase))
{
m_rehashCASContent = true;
}
else if (opt.Name.Equals("outputFile", StringComparison.OrdinalIgnoreCase) ||
opt.Name.Equals("of", StringComparison.OrdinalIgnoreCase))
{
outputFileName = opt.Value;
}
else if (opt.Name.Equals("inputAssertionListSizeDisparityMinimumFactor", StringComparison.OrdinalIgnoreCase) ||
opt.Name.Equals("mf", StringComparison.OrdinalIgnoreCase))
{
if (!double.TryParse(opt.Value, out m_inputAssertionListSizeDisparityMinimumFactor))
{
throw Error("The value for option '" + opt.Name + "' must be a valid double");
}
if (m_inputAssertionListSizeDisparityMinimumFactor <= 1.0)
{
WriteWarning("WARNING! The value for option '" + opt.Name + "' must be > 1" +
". Defaulting to " + InputAssertionListChecker.DefaultDisparityFactor);
m_inputAssertionListSizeDisparityMinimumFactor = InputAssertionListChecker.DefaultDisparityFactor;
}
}
else if (opt.Name.Equals("weakFingerprintInputFilepath", StringComparison.OrdinalIgnoreCase) ||
opt.Name.Equals("wi", StringComparison.OrdinalIgnoreCase))
{
weakFingerprintInputFilepath = opt.Value;
}
else if (opt.Name.Equals("weakFingerprintOutputFilepath", StringComparison.OrdinalIgnoreCase) ||
opt.Name.Equals("wo", StringComparison.OrdinalIgnoreCase))
{
m_weakFingerprintOutputFilepath = opt.Value;
}
else if (opt.Name.Equals("inputAssertionListDump", StringComparison.OrdinalIgnoreCase) ||
opt.Name.Equals("id", StringComparison.OrdinalIgnoreCase))
{
m_dumpInputAssertionLists = true;
}
else if (opt.Name.Equals("inputAssertionListDumpMustIncludeRegex", StringComparison.OrdinalIgnoreCase) ||
opt.Name.Equals("ii", StringComparison.OrdinalIgnoreCase))
{
inputAssertionListDumpMustIncludeRegex = opt.Value;
}
else if (opt.Name.Equals("inputAssertionListDumpMustNotIncludeRegex", StringComparison.OrdinalIgnoreCase) ||
opt.Name.Equals("in", StringComparison.OrdinalIgnoreCase))
{
inputAssertionListDumpMustNotIncludeRegex = opt.Value;
}
else
{
WriteWarning("WARNING! Unrecognized command line option: " + opt.Name);
}
}
// Initialize json config string
bool jsonFileProvided = jsonFilePath != null;
if (jsonFileProvided)
{
m_jsonString = File.ReadAllText(jsonFilePath);
}
if (string.IsNullOrEmpty(m_jsonString))
{
throw Error("Either the json string must be specified or a path to a file containing the json string must be specified.");
}
// Initialize session id regex
if (string.IsNullOrEmpty(sessionIDFilter))
{
m_sessionRegex = new Regex(".*", RegexOptions.Compiled);
}
else
{
try
{
m_sessionRegex = new Regex(sessionIDFilter, RegexOptions.Compiled | RegexOptions.CultureInvariant);
}
catch (Exception e)
{
throw Error("Initializing the session ID regex failed with exception: [{0}].", e);
}
}
if (string.IsNullOrEmpty(inputAssertionListDumpMustIncludeRegex))
{
m_inputAssertionListDumpMustIncludeRegex = null;
}
else
{
try
{
m_inputAssertionListDumpMustIncludeRegex = new Regex(inputAssertionListDumpMustIncludeRegex, RegexOptions.Compiled | RegexOptions.CultureInvariant);
}
catch (Exception e)
{
throw Error("Initializing the input assertion list must include regex failed with exception: [{0}].", e);
}
}
if (string.IsNullOrEmpty(inputAssertionListDumpMustNotIncludeRegex))
{
m_inputAssertionListDumpMustNotIncludeRegex = null;
}
else
{
try
{
m_inputAssertionListDumpMustNotIncludeRegex = new Regex(inputAssertionListDumpMustNotIncludeRegex, RegexOptions.Compiled | RegexOptions.CultureInvariant);
}
catch (Exception e)
{
throw Error("Initializing the input assertion list must not include regex failed with exception: [{0}].", e);
}
}
// Initialize output destination text writer
if (string.IsNullOrEmpty(outputFileName))
{
m_outputDestination = Console.Out;
m_outputBasePath = null;
Console.Error.WriteLine("\nUsing output destination: Console");
}
else
{
m_outputBasePath = Path.GetDirectoryName(outputFileName);
FileStream fileStream = null;
try
{
fileStream = new FileStream(outputFileName, FileMode.Create);
m_outputDestination = new StreamWriter(fileStream);
}
catch (Exception e)
{
if (fileStream != null)
{
fileStream.Dispose();
}
throw Error("Opening the output file failed with exception: [{0}]", e);
}
Console.Error.WriteLine("\nUsing output destination: " + outputFileName);
}
// Read in weak fingerprints to use
// Expected file format is one weak fingerprint per line, as a hex string
if (!string.IsNullOrEmpty(weakFingerprintInputFilepath))
{
FileStream fileStream = File.OpenRead(weakFingerprintInputFilepath);
try
{
using (StreamReader streamReader = new StreamReader(fileStream))
{
fileStream = null;
WeakFingerprintHash weakFingerprint;
string line;
while ((line = streamReader.ReadLine()) != null)
{
if (WeakFingerprintHash.TryParse(line, out weakFingerprint))
{
m_inputWeakFingerprints.Add(weakFingerprint);
}
}
}
}
finally
{
if (fileStream != null)
{
fileStream.Dispose();
}
}
if (m_inputWeakFingerprints.Count == 0)
{
throw Error("Could not load any weak fingerprints from {0}", weakFingerprintInputFilepath);
}
}
m_weakFingerprintsFound = string.IsNullOrEmpty(m_weakFingerprintOutputFilepath) ? null : new ConcurrentDictionary <WeakFingerprintHash, byte>();
}
public async Task <Possible <FullCacheRecordWithDeterminism, Failure> > AddOrGetAsync(WeakFingerprintHash weak, CasHash casElement, Hash hashElement, CasEntries hashes, UrgencyHint urgencyHint, Guid activityId)
{
Contract.Requires(!IsClosed);
Contract.Requires(hashes.IsValid);
Contract.Assert(!IsReadOnly);
using (var counter = m_counters.AddOrGetCounter())
{
using (var eventing = new AddOrGetActivity(BasicFilesystemCache.EventSource, activityId, this))
{
eventing.Start(weak, casElement, hashElement, hashes, urgencyHint);
counter.SetEntriesCount(hashes.Count); // The size of what we are adding (effectively)
// We check the Cas entries if we are strict
if (StrictMetadataCasCoupling)
{
// Check that the content is valid.
if (!m_pinnedToCas.ContainsKey(casElement))
{
counter.Failed();
return(eventing.StopFailure(new UnpinnedCasEntryFailure(CacheId, casElement)));
}
foreach (CasHash hash in hashes)
{
if (!m_pinnedToCas.ContainsKey(hash))
{
counter.Failed();
return(eventing.StopFailure(new UnpinnedCasEntryFailure(CacheId, hash)));
}
}
}
StrongFingerprint strong = new StrongFingerprint(weak, casElement, hashElement, CacheId);
// Assume we accepted the Add and there is nothing to return
FullCacheRecord result = null;
string strongFingerprintName = m_cache.GetStrongFingerprintFilename(strong);
try
{
using (FileStream file = await m_cache.ContendedOpenStreamAsync(strongFingerprintName, FileMode.OpenOrCreate, FileAccess.ReadWrite))
{
// The compiler thinks that it is not assigned down below at the end
// even though it would be set by the writeEntries being true in that case
CasEntries oldCasEntries = hashes;
// Assume we will write our new enties to the file.
bool writeEntries = true;
// If there is some data in the file already, we need to try to read it.
if (file.Length > 0)
{
var possibleOldCasEntries = await m_cache.ReadCacheEntryAsync(file);
// Only if it was formatted correctly do we continue to check if
// we should replace it.
if (possibleOldCasEntries.Succeeded)
{
oldCasEntries = possibleOldCasEntries.Result;
writeEntries = false;
// We can only replace if both or neither is SinglePhaseNonDeterministic
if (oldCasEntries.Determinism.IsSinglePhaseNonDeterministic != hashes.Determinism.IsSinglePhaseNonDeterministic)
{
counter.Failed();
return(eventing.StopFailure(new SinglePhaseMixingFailure(CacheId)));
}
// Should we replace?
if (hashes.Determinism.IsSinglePhaseNonDeterministic ||
(!oldCasEntries.Determinism.IsDeterministicTool &&
(hashes.Determinism.IsDeterministic && !oldCasEntries.Determinism.Equals(hashes.Determinism))))
{
// We are replacing due to determinism
counter.Det();
writeEntries = true;
}
else if (HasMissingContent(oldCasEntries))
{
counter.Repair();
writeEntries = true;
}
else if (oldCasEntries.Determinism.IsDeterministicTool && hashes.Determinism.IsDeterministicTool && !oldCasEntries.Equals(hashes))
{
// We have a non-deterministic tool!
counter.Failed();
return(eventing.StopFailure(new NotDeterministicFailure(CacheId, new FullCacheRecord(strong, oldCasEntries), new FullCacheRecord(strong, hashes))));
}
}
}
// Are we going to write the entries?
if (writeEntries)
{
// We are writing so the old entries don't count
oldCasEntries = hashes;
// Write from the front
file.SetLength(0);
await m_cache.WriteCacheEntryAsync(file, hashes);
}
else
{
counter.Dup();
}
// If what is in the cache is different than what we are
// asking to add, build a FullCacheRecord to return what
// is in the cache.
if (!oldCasEntries.Equals(hashes))
{
counter.Get();
result = new FullCacheRecord(strong, oldCasEntries);
}
}
}
catch (Exception e)
{
counter.Failed();
return(eventing.StopFailure(new StrongFingerprintAccessFailure(m_cache.CacheId, strong, e)));
}
AddSessionRecord(strong);
if (result != null)
{
return
(eventing.Returns(
new FullCacheRecordWithDeterminism(
new FullCacheRecord(
result.StrongFingerprint,
result.CasEntries.GetModifiedCasEntriesWithDeterminism(
m_cache.IsAuthoritative,
m_cache.CacheGuid,
CacheDeterminism.NeverExpires)))));
}
else
{
return(eventing.Returns(new FullCacheRecordWithDeterminism(hashes.GetFinalDeterminism(m_cache.IsAuthoritative, m_cache.CacheGuid, DateTime.UtcNow.Add(m_cache.TimeToLive)))));
}
}
}
}
