private async Task ValidateSessionAsync(HashSet <FullCacheRecord> expectedRecords, ICache cache, string sessionId, FakeBuild.CasAccessMethod accessMethod) { if (!ImplementsTrackedSessions || DummySessionName != null) { return; } ICacheReadOnlySession readOnlySession = (await cache.CreateReadOnlySessionAsync()).Success(); // Check that the content is fine HashSet <FullCacheRecord> foundRecords = new HashSet <FullCacheRecord>(); foreach (var strongFingerprintTask in cache.EnumerateSessionStrongFingerprints(DummySessionName ?? sessionId).Success().OutOfOrderTasks()) { StrongFingerprint strongFingerprint = await strongFingerprintTask; CasEntries casEntries = (await readOnlySession.GetCacheEntryAsync(strongFingerprint)).Success(); FullCacheRecord record = new FullCacheRecord(strongFingerprint, casEntries); XAssert.IsTrue(expectedRecords.Contains(record), "Found record that was not expected!"); foundRecords.Add(record); } (await readOnlySession.CloseAsync()).Success(); await FakeBuild.CheckContentsAsync(cache, foundRecords, accessMethod); XAssert.AreEqual(expectedRecords.Count, foundRecords.Count); }
/// <summary> /// Uses the cache instance that will be checked to generate /// a read only session to the cache /// </summary> /// <param name="cache">The cache instance to check</param> /// <param name="checkCASContent">If true, each CAS content file /// will be downloaded and rehashed to ensure that the content /// has not been corrupted since initially being put in cache</param> /// <param name="isLocalCache">Specifies whether the cache is local</param> public SingleCacheChecker(ICache cache, bool checkCASContent, bool isLocalCache = false) { m_cache = cache; m_readOnlySession = m_cache.CreateReadOnlySessionAsync().Result.Result; m_checkCASContent = checkCASContent; m_isLocalCache = isLocalCache; }
internal static IEnumerable <Task <Tuple <CasHash, long> > > GetContentSizes(this ICacheReadOnlySession session, IEnumerable <CasHash> casEntries) { foreach (var casHash in casEntries) { yield return(session.GetContentSizeAsync(casHash)); } }
/// <summary> /// Uses the specified json config string to initialize the /// cache and then uses that cache instance to generate a read /// only session to the cache /// </summary> /// <param name="cacheConfigJSONData">Json string that represents /// the cache to check</param> /// <param name="checkCASContent">If true, each CAS content file /// will be downloaded and rehashed to ensure that the content has /// not been corrupted since initially being put in cache</param> /// <param name="isLocalCache">Specifies whether the cache is local</param> public SingleCacheChecker(string cacheConfigJSONData, bool checkCASContent, bool isLocalCache = false) { m_cache = CacheFactory.InitializeCacheAsync(cacheConfigJSONData, default(Guid)).Result.Result; m_readOnlySession = m_cache.CreateReadOnlySessionAsync().Result.Result; m_checkCASContent = checkCASContent; m_isLocalCache = isLocalCache; }
/// <summary> /// Creates a read only session for the cache to check /// </summary> /// <param name="cache">The cache to check for anomalies</param> /// <param name="minDisparityFactor">This is the minimum factor of /// disparity in the size of input assertion list files that consitutes /// an anonaly</param> public InputAssertionListChecker(ICache cache, double minDisparityFactor = DefaultDisparityFactor) { Contract.Requires(cache != null); Contract.Requires(minDisparityFactor > 1.0); m_cache = cache; m_readOnlySession = m_cache.CreateReadOnlySessionAsync().Result.Result; m_minDisparityFactor = minDisparityFactor; }
internal CompositingReadOnlyCacheSession( ICacheReadOnlySession metadataSession, ICacheReadOnlySession casSesssion, ICache cache) { Cache = cache; m_metadataSession = metadataSession; m_casSession = casSesssion; PinnedToCas = new ConcurrentDictionary <CasHash, int>(); PinnedToCas.TryAdd(CasHash.NoItem, 0); }
/// <summary> /// Check the fake build via the cache given /// </summary> /// <param name="cache">The cache to read from (uses a read-only session)</param> /// <param name="index">The "index" CasHash</param> /// <param name="entries">The CasHash entries that should match the index</param> /// <param name="accessMethod">Method (File or stream) for how files are materialized from the cache</param> /// <returns>Success if all worked</returns> /// <remarks> /// This is tied to the FakeBuild where the set of results is /// made as the index file which we use in the strong fingerprint /// and basically describes the set of entries that should be in the CasEntries /// </remarks> public static async Task CheckContentsAsync(ICache cache, CasHash index, CasEntries entries, CasAccessMethod accessMethod = CasAccessMethod.Stream) { Contract.Requires(cache != null); Contract.Requires(entries.IsValid); ICacheReadOnlySession session = await cache.CreateReadOnlySessionAsync().SuccessAsync(); await CheckContentsAsync(session, index, entries, accessMethod); await session.CloseAsync().SuccessAsync(); }
/// <summary> /// Activity Constructor /// </summary> /// <param name="eventSource">The cache's event source</param> /// <param name="relatedActivityId">The related activity ID as passed to the API</param> /// <param name="cache">The ICache session instance (this)</param> public AddOrGetActivity(EventSource eventSource, Guid relatedActivityId, ICacheReadOnlySession cache) : base( eventSource, new EventSourceOptions { Level = EventLevel.Informational, Keywords = Keywords.TelemetryKeyword | Keywords.AddOrGet, }, relatedActivityId, InterfaceNames.AddOrGet, cache.CacheId) { }
private async Task <long> PinAndGetStreamSize(ICacheReadOnlySession session, CasHash hash) { long result; await session.PinToCasAsync(hash, CancellationToken.None).SuccessAsync(); using (var stream = await session.GetStreamAsync(hash).SuccessAsync()) { result = stream.Length; } return(result); }
/// <summary> /// Check that the fake build contents are available and correct /// </summary> /// <param name="cache">The cache to read from (uses a read-only session)</param> /// <param name="records">The full cache records to check</param> /// <param name="accessMethod">Method (File or stream) for how files are materialized from the cache</param> /// <returns>Success if all worked</returns> /// <remarks> /// This is tied to the FakeBuild where the set of results is /// made as the index file which we use in the strong fingerprint /// and basically describes the set of entries that should be in the CasEntries /// </remarks> public static async Task CheckContentsAsync(ICache cache, IEnumerable <FullCacheRecord> records, CasAccessMethod accessMethod = CasAccessMethod.Stream) { Contract.Requires(cache != null); Contract.Requires(records != null); ICacheReadOnlySession session = await cache.CreateReadOnlySessionAsync().SuccessAsync(); foreach (FullCacheRecord record in records) { await CheckContentsAsync(session, record.StrongFingerprint.CasElement, record.CasEntries, accessMethod); } await session.CloseAsync().SuccessAsync(); }
internal static void PoisonAllRemoteSessions(ICache cache) { Contract.Requires(cache is VerticalCacheAggregator); VerticalCacheAggregator vertCache = cache as VerticalCacheAggregator; CallbackCacheWrapper remoteCache = vertCache.RemoteCache as CallbackCacheWrapper; XAssert.IsNotNull(remoteCache); remoteCache.CreateNamedSessionAsyncCallback = async(string sessionId, ICache cacheInstance) => { ICacheSession session = await cacheInstance.CreateSessionAsync(sessionId).SuccessAsync(); CallbackCacheSessionWrapper wrappedSession = new CallbackCacheSessionWrapper(session); PoisonSession(wrappedSession); return(new BuildXL.Utilities.Possible <ICacheSession, BuildXL.Utilities.Failure>(session)); }; remoteCache.CreateSessionAsyncCallback = async(ICache cacheInstance) => { ICacheSession session = await cacheInstance.CreateSessionAsync().SuccessAsync(); CallbackCacheSessionWrapper wrappedSession = new CallbackCacheSessionWrapper(session); PoisonSession(wrappedSession); return(new BuildXL.Utilities.Possible <ICacheSession, BuildXL.Utilities.Failure>(session)); }; remoteCache.CreateReadOnlySessionAsyncCallback = async(ICache cacheInstance) => { ICacheReadOnlySession session = await cacheInstance.CreateReadOnlySessionAsync().SuccessAsync(); CallbackCacheReadOnlySessionWrapper wrappedSession = new CallbackCacheReadOnlySessionWrapper(session); PoisonROSession(wrappedSession); return(new BuildXL.Utilities.Possible <ICacheReadOnlySession, BuildXL.Utilities.Failure>(session)); }; remoteCache.CacheGuidGetCallback = (ICache wrappedcache) => { XAssert.Fail("Remote Cache was called when disconnected (CacheGuid)"); return(Guid.Empty); }; }
/// <summary> /// Check the fake build via the session given /// </summary> /// <param name="session">Session to use for the check</param> /// <param name="index">The "index" CasHash</param> /// <param name="entries">The CasHash entries that should match the index</param> /// <param name="accessMethod">Method (File or stream) for how files are materialized from the cache</param> /// <returns>An Async Task</returns> public static async Task CheckContentsAsync(ICacheReadOnlySession session, CasHash index, CasEntries entries, CasAccessMethod accessMethod = CasAccessMethod.Stream) { string cacheId = await session.PinToCasAsync(index, CancellationToken.None).SuccessAsync("Cannot pin entry {0} to cache {1}", index.ToString(), session.CacheId); string[] expected = (await GetStreamAsync(index, accessMethod, session)).Success().Stream.AsString().Split(s_splitLines, StringSplitOptions.RemoveEmptyEntries); XAssert.AreEqual(expected.Length, entries.Count, "Counts did not match from cache {0}: {1} != {2}", cacheId, expected.Length, entries.Count); for (int i = 0; i < expected.Length; i++) { string casCacheId = await session.PinToCasAsync(entries[i], CancellationToken.None).SuccessAsync(); string entry = (await GetStreamAsync(entries[i], accessMethod, session)).Success().Stream.AsString(); XAssert.AreEqual(expected[i], entry, "CasEntry {0} mismatch from cache {1}: [{2}] != [{3}]", i, casCacheId, expected[i], entry); } }
public async Task DisconnectedCacheNotQueriedForStrongFingerprints() { string testCacheId = "Disconnected"; ICache testCache = await InitializeCacheAsync(NewCache(testCacheId, false)).SuccessAsync(); PoisonAllRemoteSessions(testCache); ICacheReadOnlySession roSession = await testCache.CreateReadOnlySessionAsync().SuccessAsync(); DisconnectRemoteCache(testCache); FakeBuild fb = new FakeBuild("test", 1); foreach (var fingerprint in roSession.EnumerateStrongFingerprints(new WeakFingerprintHash(FingerprintUtilities.Hash("fingerprint").ToByteArray()))) { // Just run the enumerator, should all return. } }
// We don't pin here as we do this after having pinned everything in the PinAndGetStreamSize private async Task <long> GetAsFile(ICacheReadOnlySession session, CasHash hash) { long result; var tmpFile = FileUtilities.GetTempFileName(); try { await session.ProduceFileAsync(hash, tmpFile, FileState.ReadOnly).SuccessAsync(); result = new FileInfo(tmpFile).Length; } finally { File.Delete(tmpFile); } return(result); }
internal static async Task <Tuple <CasHash, long> > GetContentSizeAsync(this ICacheReadOnlySession session, CasHash casHash) { var possibleString = await session.PinToCasAsync(casHash); if (!possibleString.Succeeded) { return(new Tuple <CasHash, long>(casHash, (long)ContentError.UnableToPin)); } var possibleStream = await session.GetStreamAsync(casHash); if (!possibleStream.Succeeded) { return(new Tuple <CasHash, long>(casHash, (long)ContentError.UnableToStream)); } long length = possibleStream.Result.Length; possibleStream.Result.Dispose(); return(new Tuple <CasHash, long>(casHash, length)); }
public async Task CacheDataPersistedReadOnlyCache() { string testName = "CacheDataPersistedReadOnly"; ICache firstInvocation = await CreateCacheAsync(testName, true); Guid originalGuid = firstInvocation.CacheGuid; ICacheSession session = (await firstInvocation.CreateSessionAsync("sessionName")).Success(); FullCacheRecord cacheRecord = await FakeBuild.DoPipAsync(session, "PipA"); (await session.CloseAsync()).Success(); await ShutdownCacheAsync(firstInvocation, testName); ICache secondInvocation = await GetExistingCacheAsync(testName, true, true); XAssert.AreEqual(originalGuid, secondInvocation.CacheGuid, "Persistent caches: GUID should not change"); XAssert.IsTrue(secondInvocation.IsReadOnly, "This should have been a read-only cache"); ICacheReadOnlySession newSession = (await secondInvocation.CreateReadOnlySessionAsync()).Success(); int fingerprintsFound = 0; foreach (var singleFingerprint in newSession.EnumerateStrongFingerprints(cacheRecord.StrongFingerprint.WeakFingerprint)) { var sfp = (await singleFingerprint).Success(); XAssert.AreEqual(cacheRecord.StrongFingerprint, sfp, "Fingerprints must match"); fingerprintsFound++; } XAssert.AreEqual(1, fingerprintsFound, "A single instance of the fingerprint should have been found after restarting the cache."); (await newSession.CloseAsync()).Success(); await ShutdownCacheAsync(secondInvocation, testName); }
/// <summary> /// Constructs an object capable of performing statistical analysis on /// the specified cache /// </summary> /// <param name="cache">The cache to perform statistical analysis on</param> public ContentBreakdownAnalyzer(ICache cache) { m_cache = cache; m_session = m_cache.CreateReadOnlySessionAsync().Result.Result; }
/// <summary> /// Enumerates input weak fingerprints and returns all strong fingerprints /// </summary> /// <param name="weakFingerprints">Weak fingerprints to iterate over</param> /// <param name="readOnlySession">The read only session to use to /// enumerate strong fingerprints</param> /// <returns>All strong fingerprints enumerated from input weak fingerprints</returns> public static IEnumerable <StrongFingerprint> ProduceStrongFingerprints(this IEnumerable <WeakFingerprintHash> weakFingerprints, ICacheReadOnlySession readOnlySession) { HashSet <StrongFingerprint> strongFingerprintsAlreadyProduced = new HashSet <StrongFingerprint>(); foreach (WeakFingerprintHash weakFingerprint in weakFingerprints) { IEnumerable <Task <Possible <StrongFingerprint, Failure> > > possibleStrongFingerprintTasks = readOnlySession.EnumerateStrongFingerprints(weakFingerprint); foreach (Task <Possible <StrongFingerprint, Failure> > possibleStrongFingerprintTask in possibleStrongFingerprintTasks.OutOfOrderTasks()) { Possible <StrongFingerprint, Failure> possibleStrongFingerprint = possibleStrongFingerprintTask.Result; if (possibleStrongFingerprint.Succeeded && strongFingerprintsAlreadyProduced.Add(possibleStrongFingerprint.Result)) { yield return(possibleStrongFingerprint.Result); } } } }
internal static IEnumerable <Task <Possible <CasEntries, Failure> > > GetCacheEntries(this ICacheReadOnlySession session, IEnumerable <StrongFingerprint> strongFingerprints) { foreach (var strongFingerprint in strongFingerprints) { yield return(session.GetCacheEntryAsync(strongFingerprint)); } }
/// <summary> /// Activity Constructor /// </summary> /// <param name="eventSource">The cache's event source</param> /// <param name="relatedActivityId">The related activity ID as passed to the API</param> /// <param name="cache">The ICache session instance (this)</param> public EnumerateStrongFingerprintsActivity(EventSource eventSource, Guid relatedActivityId, ICacheReadOnlySession cache) : base( eventSource, new EventSourceOptions { Level = EventLevel.Informational, Keywords = Keywords.TelemetryKeyword | Keywords.EnumerateStrongFingerprints, }, relatedActivityId, InterfaceNames.EnumerateStrongFingerprints, cache.CacheId, mayTerminateEarly: true) { }
public async Task SimpleDummySession() { const string TestName = "SimpleSession"; string testCacheId = MakeCacheId(TestName); ICache cache = await CreateCacheAsync(testCacheId); // Now for the session (which we base on the cache ID) string testSessionId = "Session1-" + testCacheId; ICacheSession session = await CreateSessionAsync(cache, testSessionId); // Do the default fake build for this test (first time, no cache hit) FullCacheRecord built = await FakeBuild.DoPipAsync(session, TestName); XAssert.AreEqual(FakeBuild.NewRecordCacheId, built.CacheId, "Should have been a new cache entry!"); // Now we see if we can get back the items we think we should await CloseSessionAsync(session, testSessionId); // We need a read only session to get the CasEntries ICacheReadOnlySession readOnlySession = (await cache.CreateReadOnlySessionAsync()).Success(); // Validate that the cache contains a dummy session and it has the one cache record it needs. HashSet <FullCacheRecord> found = new HashSet <FullCacheRecord>(); foreach (var strongFingerprintTask in cache.EnumerateSessionStrongFingerprints(MemoizationStoreAdapterCache.DummySessionName).Success().OutOfOrderTasks()) { StrongFingerprint strongFingerprint = await strongFingerprintTask; CasEntries casEntries = (await readOnlySession.GetCacheEntryAsync(strongFingerprint)).Success(); FullCacheRecord record = new FullCacheRecord(strongFingerprint, casEntries); // If it is not the record we already found... if (!found.Contains(record)) { found.Add(record); } XAssert.AreEqual(1, found.Count, "There should be only 1 unique record in the session"); XAssert.AreEqual(built.StrongFingerprint.WeakFingerprint, record.StrongFingerprint.WeakFingerprint); XAssert.AreEqual(built.StrongFingerprint.CasElement, record.StrongFingerprint.CasElement); XAssert.AreEqual(built.StrongFingerprint.HashElement, record.StrongFingerprint.HashElement); XAssert.AreEqual(built.CasEntries.Count, record.CasEntries.Count, "Did not return the same number of items"); XAssert.IsTrue(record.CasEntries.Equals(built.CasEntries), "Items returned are not the same hash and/or order order"); XAssert.AreEqual(built, record); // We can not check record.CasEntries.IsDeterministic // as the cache may have determined that they are deterministic // via cache determinism recovery. } XAssert.AreEqual(1, found.Count, "There should be 1 and only 1 record in the session!"); await readOnlySession.CloseAsync().SuccessAsync(); // Check that the cache has the items in it await FakeBuild.CheckContentsAsync(cache, built); // Now redo the "build" with a cache hit testSessionId = "Session2-" + testCacheId; session = await CreateSessionAsync(cache, testSessionId); FullCacheRecord rebuilt = await FakeBuild.DoPipAsync(session, TestName); XAssert.AreEqual(built, rebuilt, "Should have been the same build!"); // We make sure we did get it from a cache rather than a manual rebuild. XAssert.AreNotEqual(built.CacheId, rebuilt.CacheId, "Should not be the same cache ID"); await CloseSessionAsync(session, testSessionId); readOnlySession = await cache.CreateReadOnlySessionAsync().SuccessAsync(); // Now that we have done the second build via a cache hit, it should produce the // same cache record as before foreach (var strongFingerprintTask in cache.EnumerateSessionStrongFingerprints(MemoizationStoreAdapterCache.DummySessionName).Success().OutOfOrderTasks()) { StrongFingerprint strongFingerprint = await strongFingerprintTask; CasEntries casEntries = (await readOnlySession.GetCacheEntryAsync(strongFingerprint)).Success(); FullCacheRecord record = new FullCacheRecord(strongFingerprint, casEntries); XAssert.IsTrue(found.Contains(record), "Second session should produce the same cache record but did not!"); } (await readOnlySession.CloseAsync()).Success(); await ShutdownCacheAsync(cache, testCacheId); }
internal InputListFilterReadOnlyCacheSession(ICacheReadOnlySession session, InputListFilterCache cache) { Cache = cache; m_session = session; }
public CallbackCacheReadOnlySessionWrapper(ICacheReadOnlySession realCache) { m_realSession = realCache; }
/// <summary> /// Gets a file stream from the CAS either directly or by materlializing a file in the temp path and then opening it for read. /// </summary> /// <param name="hash">Hash for CAS entry</param> /// <param name="method">Method used to access CAS</param> /// <param name="session">Cache session</param> /// <returns>A stream pointing to the file contents, or a failure.</returns> private static async Task <Possible <StreamWithLength, Failure> > GetStreamAsync(CasHash hash, CasAccessMethod method, ICacheReadOnlySession session) { switch (method) { case CasAccessMethod.Stream: return(await session.GetStreamAsync(hash)); case CasAccessMethod.FileSystem: string filePath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); string placedFilePath = await session.ProduceFileAsync(hash, filePath, FileState.ReadOnly).SuccessAsync(); XAssert.AreEqual(filePath, placedFilePath); FileStream fs = new FileStream(placedFilePath, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.Read); File.Delete(placedFilePath); return(fs.WithLength()); default: throw new NotImplementedException(); } }
/// <summary> /// Constructs an object capable of performing statistical analysis on /// the specified cache /// </summary> /// <param name="cache">The cache to perform statistical analysis on</param> public StatisticalAnalyzer(ICache cache) { m_cache = cache; m_session = m_cache.CreateReadOnlySessionAsync().Result.Result; }