private async Task ValidateSessionAsync(HashSet <FullCacheRecord> expectedRecords, ICache cache, string sessionId, FakeBuild.CasAccessMethod accessMethod)
{
if (!ImplementsTrackedSessions || DummySessionName != null)
{
return;
}
ICacheReadOnlySession readOnlySession = (await cache.CreateReadOnlySessionAsync()).Success();
// Check that the content is fine
HashSet <FullCacheRecord> foundRecords = new HashSet <FullCacheRecord>();
foreach (var strongFingerprintTask in cache.EnumerateSessionStrongFingerprints(DummySessionName ?? sessionId).Success().OutOfOrderTasks())
{
StrongFingerprint strongFingerprint = await strongFingerprintTask;
CasEntries casEntries = (await readOnlySession.GetCacheEntryAsync(strongFingerprint)).Success();
FullCacheRecord record = new FullCacheRecord(strongFingerprint, casEntries);
XAssert.IsTrue(expectedRecords.Contains(record), "Found record that was not expected!");
foundRecords.Add(record);
}
(await readOnlySession.CloseAsync()).Success();
await FakeBuild.CheckContentsAsync(cache, foundRecords, accessMethod);
XAssert.AreEqual(expectedRecords.Count, foundRecords.Count);
}
/// <summary>
/// Uses the cache instance that will be checked to generate
/// a read only session to the cache
/// </summary>
/// <param name="cache">The cache instance to check</param>
/// <param name="checkCASContent">If true, each CAS content file
/// will be downloaded and rehashed to ensure that the content
/// has not been corrupted since initially being put in cache</param>
/// <param name="isLocalCache">Specifies whether the cache is local</param>
public SingleCacheChecker(ICache cache, bool checkCASContent, bool isLocalCache = false)
{
m_cache = cache;
m_readOnlySession = m_cache.CreateReadOnlySessionAsync().Result.Result;
m_checkCASContent = checkCASContent;
m_isLocalCache = isLocalCache;
}
internal static IEnumerable <Task <Tuple <CasHash, long> > > GetContentSizes(this ICacheReadOnlySession session, IEnumerable <CasHash> casEntries)
{
foreach (var casHash in casEntries)
{
yield return(session.GetContentSizeAsync(casHash));
}
}
/// <summary>
/// Uses the specified json config string to initialize the
/// cache and then uses that cache instance to generate a read
/// only session to the cache
/// </summary>
/// <param name="cacheConfigJSONData">Json string that represents
/// the cache to check</param>
/// <param name="checkCASContent">If true, each CAS content file
/// will be downloaded and rehashed to ensure that the content has
/// not been corrupted since initially being put in cache</param>
/// <param name="isLocalCache">Specifies whether the cache is local</param>
public SingleCacheChecker(string cacheConfigJSONData, bool checkCASContent, bool isLocalCache = false)
{
m_cache = CacheFactory.InitializeCacheAsync(cacheConfigJSONData, default(Guid)).Result.Result;
m_readOnlySession = m_cache.CreateReadOnlySessionAsync().Result.Result;
m_checkCASContent = checkCASContent;
m_isLocalCache = isLocalCache;
}
/// <summary>
/// Creates a read only session for the cache to check
/// </summary>
/// <param name="cache">The cache to check for anomalies</param>
/// <param name="minDisparityFactor">This is the minimum factor of
/// disparity in the size of input assertion list files that consitutes
/// an anonaly</param>
public InputAssertionListChecker(ICache cache, double minDisparityFactor = DefaultDisparityFactor)
{
Contract.Requires(cache != null);
Contract.Requires(minDisparityFactor > 1.0);
m_cache = cache;
m_readOnlySession = m_cache.CreateReadOnlySessionAsync().Result.Result;
m_minDisparityFactor = minDisparityFactor;
}
internal CompositingReadOnlyCacheSession(
ICacheReadOnlySession metadataSession,
ICacheReadOnlySession casSesssion,
ICache cache)
{
Cache = cache;
m_metadataSession = metadataSession;
m_casSession = casSesssion;
PinnedToCas = new ConcurrentDictionary <CasHash, int>();
PinnedToCas.TryAdd(CasHash.NoItem, 0);
}
/// <summary>
/// Check the fake build via the cache given
/// </summary>
/// <param name="cache">The cache to read from (uses a read-only session)</param>
/// <param name="index">The "index" CasHash</param>
/// <param name="entries">The CasHash entries that should match the index</param>
/// <param name="accessMethod">Method (File or stream) for how files are materialized from the cache</param>
/// <returns>Success if all worked</returns>
/// <remarks>
/// This is tied to the FakeBuild where the set of results is
/// made as the index file which we use in the strong fingerprint
/// and basically describes the set of entries that should be in the CasEntries
/// </remarks>
public static async Task CheckContentsAsync(ICache cache, CasHash index, CasEntries entries, CasAccessMethod accessMethod = CasAccessMethod.Stream)
{
Contract.Requires(cache != null);
Contract.Requires(entries.IsValid);
ICacheReadOnlySession session = await cache.CreateReadOnlySessionAsync().SuccessAsync();
await CheckContentsAsync(session, index, entries, accessMethod);
await session.CloseAsync().SuccessAsync();
}
/// <summary>
/// Activity Constructor
/// </summary>
/// <param name="eventSource">The cache's event source</param>
/// <param name="relatedActivityId">The related activity ID as passed to the API</param>
/// <param name="cache">The ICache session instance (this)</param>
public AddOrGetActivity(EventSource eventSource, Guid relatedActivityId, ICacheReadOnlySession cache)
: base(
eventSource,
new EventSourceOptions
{
Level = EventLevel.Informational,
Keywords = Keywords.TelemetryKeyword | Keywords.AddOrGet,
},
relatedActivityId,
InterfaceNames.AddOrGet,
cache.CacheId)
{
}
private async Task <long> PinAndGetStreamSize(ICacheReadOnlySession session, CasHash hash)
{
long result;
await session.PinToCasAsync(hash, CancellationToken.None).SuccessAsync();
using (var stream = await session.GetStreamAsync(hash).SuccessAsync())
{
result = stream.Length;
}
return(result);
}
/// <summary>
/// Check that the fake build contents are available and correct
/// </summary>
/// <param name="cache">The cache to read from (uses a read-only session)</param>
/// <param name="records">The full cache records to check</param>
/// <param name="accessMethod">Method (File or stream) for how files are materialized from the cache</param>
/// <returns>Success if all worked</returns>
/// <remarks>
/// This is tied to the FakeBuild where the set of results is
/// made as the index file which we use in the strong fingerprint
/// and basically describes the set of entries that should be in the CasEntries
/// </remarks>
public static async Task CheckContentsAsync(ICache cache, IEnumerable <FullCacheRecord> records, CasAccessMethod accessMethod = CasAccessMethod.Stream)
{
Contract.Requires(cache != null);
Contract.Requires(records != null);
ICacheReadOnlySession session = await cache.CreateReadOnlySessionAsync().SuccessAsync();
foreach (FullCacheRecord record in records)
{
await CheckContentsAsync(session, record.StrongFingerprint.CasElement, record.CasEntries, accessMethod);
}
await session.CloseAsync().SuccessAsync();
}
internal static void PoisonAllRemoteSessions(ICache cache)
{
Contract.Requires(cache is VerticalCacheAggregator);
VerticalCacheAggregator vertCache = cache as VerticalCacheAggregator;
CallbackCacheWrapper remoteCache = vertCache.RemoteCache as CallbackCacheWrapper;
XAssert.IsNotNull(remoteCache);
remoteCache.CreateNamedSessionAsyncCallback = async(string sessionId, ICache cacheInstance) =>
{
ICacheSession session = await cacheInstance.CreateSessionAsync(sessionId).SuccessAsync();
CallbackCacheSessionWrapper wrappedSession = new CallbackCacheSessionWrapper(session);
PoisonSession(wrappedSession);
return(new BuildXL.Utilities.Possible <ICacheSession, BuildXL.Utilities.Failure>(session));
};
remoteCache.CreateSessionAsyncCallback = async(ICache cacheInstance) =>
{
ICacheSession session = await cacheInstance.CreateSessionAsync().SuccessAsync();
CallbackCacheSessionWrapper wrappedSession = new CallbackCacheSessionWrapper(session);
PoisonSession(wrappedSession);
return(new BuildXL.Utilities.Possible <ICacheSession, BuildXL.Utilities.Failure>(session));
};
remoteCache.CreateReadOnlySessionAsyncCallback = async(ICache cacheInstance) =>
{
ICacheReadOnlySession session = await cacheInstance.CreateReadOnlySessionAsync().SuccessAsync();
CallbackCacheReadOnlySessionWrapper wrappedSession = new CallbackCacheReadOnlySessionWrapper(session);
PoisonROSession(wrappedSession);
return(new BuildXL.Utilities.Possible <ICacheReadOnlySession, BuildXL.Utilities.Failure>(session));
};
remoteCache.CacheGuidGetCallback = (ICache wrappedcache) =>
{
XAssert.Fail("Remote Cache was called when disconnected (CacheGuid)");
return(Guid.Empty);
};
}
/// <summary>
/// Check the fake build via the session given
/// </summary>
/// <param name="session">Session to use for the check</param>
/// <param name="index">The "index" CasHash</param>
/// <param name="entries">The CasHash entries that should match the index</param>
/// <param name="accessMethod">Method (File or stream) for how files are materialized from the cache</param>
/// <returns>An Async Task</returns>
public static async Task CheckContentsAsync(ICacheReadOnlySession session, CasHash index, CasEntries entries, CasAccessMethod accessMethod = CasAccessMethod.Stream)
{
string cacheId = await session.PinToCasAsync(index, CancellationToken.None).SuccessAsync("Cannot pin entry {0} to cache {1}", index.ToString(), session.CacheId);
string[] expected = (await GetStreamAsync(index, accessMethod, session)).Success().Stream.AsString().Split(s_splitLines, StringSplitOptions.RemoveEmptyEntries);
XAssert.AreEqual(expected.Length, entries.Count, "Counts did not match from cache {0}: {1} != {2}", cacheId, expected.Length, entries.Count);
for (int i = 0; i < expected.Length; i++)
{
string casCacheId = await session.PinToCasAsync(entries[i], CancellationToken.None).SuccessAsync();
string entry = (await GetStreamAsync(entries[i], accessMethod, session)).Success().Stream.AsString();
XAssert.AreEqual(expected[i], entry, "CasEntry {0} mismatch from cache {1}: [{2}] != [{3}]", i, casCacheId, expected[i], entry);
}
}
public async Task DisconnectedCacheNotQueriedForStrongFingerprints()
{
string testCacheId = "Disconnected";
ICache testCache = await InitializeCacheAsync(NewCache(testCacheId, false)).SuccessAsync();
PoisonAllRemoteSessions(testCache);
ICacheReadOnlySession roSession = await testCache.CreateReadOnlySessionAsync().SuccessAsync();
DisconnectRemoteCache(testCache);
FakeBuild fb = new FakeBuild("test", 1);
foreach (var fingerprint in roSession.EnumerateStrongFingerprints(new WeakFingerprintHash(FingerprintUtilities.Hash("fingerprint").ToByteArray())))
{
// Just run the enumerator, should all return.
}
}
// We don't pin here as we do this after having pinned everything in the PinAndGetStreamSize
private async Task <long> GetAsFile(ICacheReadOnlySession session, CasHash hash)
{
long result;
var tmpFile = FileUtilities.GetTempFileName();
try
{
await session.ProduceFileAsync(hash, tmpFile, FileState.ReadOnly).SuccessAsync();
result = new FileInfo(tmpFile).Length;
}
finally
{
File.Delete(tmpFile);
}
return(result);
}
internal static async Task <Tuple <CasHash, long> > GetContentSizeAsync(this ICacheReadOnlySession session, CasHash casHash)
{
var possibleString = await session.PinToCasAsync(casHash);
if (!possibleString.Succeeded)
{
return(new Tuple <CasHash, long>(casHash, (long)ContentError.UnableToPin));
}
var possibleStream = await session.GetStreamAsync(casHash);
if (!possibleStream.Succeeded)
{
return(new Tuple <CasHash, long>(casHash, (long)ContentError.UnableToStream));
}
long length = possibleStream.Result.Length;
possibleStream.Result.Dispose();
return(new Tuple <CasHash, long>(casHash, length));
}
public async Task CacheDataPersistedReadOnlyCache()
{
string testName = "CacheDataPersistedReadOnly";
ICache firstInvocation = await CreateCacheAsync(testName, true);
Guid originalGuid = firstInvocation.CacheGuid;
ICacheSession session = (await firstInvocation.CreateSessionAsync("sessionName")).Success();
FullCacheRecord cacheRecord = await FakeBuild.DoPipAsync(session, "PipA");
(await session.CloseAsync()).Success();
await ShutdownCacheAsync(firstInvocation, testName);
ICache secondInvocation = await GetExistingCacheAsync(testName, true, true);
XAssert.AreEqual(originalGuid, secondInvocation.CacheGuid, "Persistent caches: GUID should not change");
XAssert.IsTrue(secondInvocation.IsReadOnly, "This should have been a read-only cache");
ICacheReadOnlySession newSession = (await secondInvocation.CreateReadOnlySessionAsync()).Success();
int fingerprintsFound = 0;
foreach (var singleFingerprint in newSession.EnumerateStrongFingerprints(cacheRecord.StrongFingerprint.WeakFingerprint))
{
var sfp = (await singleFingerprint).Success();
XAssert.AreEqual(cacheRecord.StrongFingerprint, sfp, "Fingerprints must match");
fingerprintsFound++;
}
XAssert.AreEqual(1, fingerprintsFound, "A single instance of the fingerprint should have been found after restarting the cache.");
(await newSession.CloseAsync()).Success();
await ShutdownCacheAsync(secondInvocation, testName);
}
/// <summary>
/// Constructs an object capable of performing statistical analysis on
/// the specified cache
/// </summary>
/// <param name="cache">The cache to perform statistical analysis on</param>
public ContentBreakdownAnalyzer(ICache cache)
{
m_cache = cache;
m_session = m_cache.CreateReadOnlySessionAsync().Result.Result;
}
/// <summary>
/// Enumerates input weak fingerprints and returns all strong fingerprints
/// </summary>
/// <param name="weakFingerprints">Weak fingerprints to iterate over</param>
/// <param name="readOnlySession">The read only session to use to
/// enumerate strong fingerprints</param>
/// <returns>All strong fingerprints enumerated from input weak fingerprints</returns>
public static IEnumerable <StrongFingerprint> ProduceStrongFingerprints(this IEnumerable <WeakFingerprintHash> weakFingerprints, ICacheReadOnlySession readOnlySession)
{
HashSet <StrongFingerprint> strongFingerprintsAlreadyProduced = new HashSet <StrongFingerprint>();
foreach (WeakFingerprintHash weakFingerprint in weakFingerprints)
{
IEnumerable <Task <Possible <StrongFingerprint, Failure> > > possibleStrongFingerprintTasks = readOnlySession.EnumerateStrongFingerprints(weakFingerprint);
foreach (Task <Possible <StrongFingerprint, Failure> > possibleStrongFingerprintTask in possibleStrongFingerprintTasks.OutOfOrderTasks())
{
Possible <StrongFingerprint, Failure> possibleStrongFingerprint = possibleStrongFingerprintTask.Result;
if (possibleStrongFingerprint.Succeeded && strongFingerprintsAlreadyProduced.Add(possibleStrongFingerprint.Result))
{
yield return(possibleStrongFingerprint.Result);
}
}
}
}
internal static IEnumerable <Task <Possible <CasEntries, Failure> > > GetCacheEntries(this ICacheReadOnlySession session, IEnumerable <StrongFingerprint> strongFingerprints)
{
foreach (var strongFingerprint in strongFingerprints)
{
yield return(session.GetCacheEntryAsync(strongFingerprint));
}
}
/// <summary>
/// Activity Constructor
/// </summary>
/// <param name="eventSource">The cache's event source</param>
/// <param name="relatedActivityId">The related activity ID as passed to the API</param>
/// <param name="cache">The ICache session instance (this)</param>
public EnumerateStrongFingerprintsActivity(EventSource eventSource, Guid relatedActivityId, ICacheReadOnlySession cache)
: base(
eventSource,
new EventSourceOptions
{
Level = EventLevel.Informational,
Keywords = Keywords.TelemetryKeyword | Keywords.EnumerateStrongFingerprints,
},
relatedActivityId,
InterfaceNames.EnumerateStrongFingerprints,
cache.CacheId,
mayTerminateEarly: true)
{
}
public async Task SimpleDummySession()
{
const string TestName = "SimpleSession";
string testCacheId = MakeCacheId(TestName);
ICache cache = await CreateCacheAsync(testCacheId);
// Now for the session (which we base on the cache ID)
string testSessionId = "Session1-" + testCacheId;
ICacheSession session = await CreateSessionAsync(cache, testSessionId);
// Do the default fake build for this test (first time, no cache hit)
FullCacheRecord built = await FakeBuild.DoPipAsync(session, TestName);
XAssert.AreEqual(FakeBuild.NewRecordCacheId, built.CacheId, "Should have been a new cache entry!");
// Now we see if we can get back the items we think we should
await CloseSessionAsync(session, testSessionId);
// We need a read only session to get the CasEntries
ICacheReadOnlySession readOnlySession = (await cache.CreateReadOnlySessionAsync()).Success();
// Validate that the cache contains a dummy session and it has the one cache record it needs.
HashSet <FullCacheRecord> found = new HashSet <FullCacheRecord>();
foreach (var strongFingerprintTask in cache.EnumerateSessionStrongFingerprints(MemoizationStoreAdapterCache.DummySessionName).Success().OutOfOrderTasks())
{
StrongFingerprint strongFingerprint = await strongFingerprintTask;
CasEntries casEntries = (await readOnlySession.GetCacheEntryAsync(strongFingerprint)).Success();
FullCacheRecord record = new FullCacheRecord(strongFingerprint, casEntries);
// If it is not the record we already found...
if (!found.Contains(record))
{
found.Add(record);
}
XAssert.AreEqual(1, found.Count, "There should be only 1 unique record in the session");
XAssert.AreEqual(built.StrongFingerprint.WeakFingerprint, record.StrongFingerprint.WeakFingerprint);
XAssert.AreEqual(built.StrongFingerprint.CasElement, record.StrongFingerprint.CasElement);
XAssert.AreEqual(built.StrongFingerprint.HashElement, record.StrongFingerprint.HashElement);
XAssert.AreEqual(built.CasEntries.Count, record.CasEntries.Count, "Did not return the same number of items");
XAssert.IsTrue(record.CasEntries.Equals(built.CasEntries), "Items returned are not the same hash and/or order order");
XAssert.AreEqual(built, record);
// We can not check record.CasEntries.IsDeterministic
// as the cache may have determined that they are deterministic
// via cache determinism recovery.
}
XAssert.AreEqual(1, found.Count, "There should be 1 and only 1 record in the session!");
await readOnlySession.CloseAsync().SuccessAsync();
// Check that the cache has the items in it
await FakeBuild.CheckContentsAsync(cache, built);
// Now redo the "build" with a cache hit
testSessionId = "Session2-" + testCacheId;
session = await CreateSessionAsync(cache, testSessionId);
FullCacheRecord rebuilt = await FakeBuild.DoPipAsync(session, TestName);
XAssert.AreEqual(built, rebuilt, "Should have been the same build!");
// We make sure we did get it from a cache rather than a manual rebuild.
XAssert.AreNotEqual(built.CacheId, rebuilt.CacheId, "Should not be the same cache ID");
await CloseSessionAsync(session, testSessionId);
readOnlySession = await cache.CreateReadOnlySessionAsync().SuccessAsync();
// Now that we have done the second build via a cache hit, it should produce the
// same cache record as before
foreach (var strongFingerprintTask in cache.EnumerateSessionStrongFingerprints(MemoizationStoreAdapterCache.DummySessionName).Success().OutOfOrderTasks())
{
StrongFingerprint strongFingerprint = await strongFingerprintTask;
CasEntries casEntries = (await readOnlySession.GetCacheEntryAsync(strongFingerprint)).Success();
FullCacheRecord record = new FullCacheRecord(strongFingerprint, casEntries);
XAssert.IsTrue(found.Contains(record), "Second session should produce the same cache record but did not!");
}
(await readOnlySession.CloseAsync()).Success();
await ShutdownCacheAsync(cache, testCacheId);
}
internal InputListFilterReadOnlyCacheSession(ICacheReadOnlySession session, InputListFilterCache cache)
{
Cache = cache;
m_session = session;
}
public CallbackCacheReadOnlySessionWrapper(ICacheReadOnlySession realCache)
{
m_realSession = realCache;
}
/// <summary>
/// Gets a file stream from the CAS either directly or by materlializing a file in the temp path and then opening it for read.
/// </summary>
/// <param name="hash">Hash for CAS entry</param>
/// <param name="method">Method used to access CAS</param>
/// <param name="session">Cache session</param>
/// <returns>A stream pointing to the file contents, or a failure.</returns>
private static async Task <Possible <StreamWithLength, Failure> > GetStreamAsync(CasHash hash, CasAccessMethod method, ICacheReadOnlySession session)
{
switch (method)
{
case CasAccessMethod.Stream:
return(await session.GetStreamAsync(hash));
case CasAccessMethod.FileSystem:
string filePath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
string placedFilePath = await session.ProduceFileAsync(hash, filePath, FileState.ReadOnly).SuccessAsync();
XAssert.AreEqual(filePath, placedFilePath);
FileStream fs = new FileStream(placedFilePath, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.Read);
File.Delete(placedFilePath);
return(fs.WithLength());
default:
throw new NotImplementedException();
}
}
/// <summary>
/// Constructs an object capable of performing statistical analysis on
/// the specified cache
/// </summary>
/// <param name="cache">The cache to perform statistical analysis on</param>
public StatisticalAnalyzer(ICache cache)
{
m_cache = cache;
m_session = m_cache.CreateReadOnlySessionAsync().Result.Result;
}
