public async Task TestGcCorruptedFingerprint() { string cacheConfig = TestType.NewCache(nameof(TestGcCorruptedFingerprint), true); BasicFilesystemCache cache = (await InitializeCacheAsync(cacheConfig).SuccessAsync()) as BasicFilesystemCache; XAssert.IsNotNull(cache, "Failed to create cache for GC tests!"); PipDefinition[] pips = { new PipDefinition("Pip1"), new PipDefinition("Pip2"), new PipDefinition("Pip3") }; CacheEntries files = await BuildPipsAndGetCacheEntries(cache, "Build", pips); // We will corrupt one of the fingerprints var fp = files.FingerprintFiles.Values.ToArray(); // Corrupt the fingerprint by keeping the same size but writing all zeros // Which is a common corruption on hardware/OS resets. long length = fp[0].Length; using (FileStream fs = fp[0].OpenWrite()) { while (length-- > 0) { fs.WriteByte(0); } } // Corrupt the second fingerprint by trimming the file length using (FileStream fs = fp[1].OpenWrite()) { fs.SetLength(0); } // Now do a GC that reads the fingerprints (collecting the CAS items) Dictionary <string, double> statsCas = cache.CollectUnreferencedCasItems(m_output).Success("{0}:CollectUnreferencedCasItems - corrupted fingerprint"); m_output.LogStats(statsCas); // There should now be only 1 valid fingerprint and 2 different // invalid ones. XAssert.AreEqual(statsCas["CAS_ReadRoots_Fingerprints"], 1); XAssert.AreEqual(statsCas["CAS_ReadRoots_InvalidFingerprints"], 2); // Check that the partition counts match as needed XAssert.AreEqual((TestType is TestBasicFilesystemSharded) ? TestBasicFilesystemSharded.SHARD_COUNT : 1, statsCas["CAS_Partitions"]); AssertSuccess(await cache.ShutdownAsync()); }
private Dictionary <string, double> FullGc(BasicFilesystemCache cache, string action = "FullGc") { Dictionary <string, double> statsFp = cache.CollectUnreferencedFingerprints(m_output).Success("{0}:CollectUnreferencedFingerprints\n{1}", action); Dictionary <string, double> statsCas = cache.CollectUnreferencedCasItems(m_output).Success("{0}:CollectUnreferencedCasItems\n{1}", action); foreach (var kv in statsCas) { statsFp.Add(kv.Key, kv.Value); } m_output.LogStats(statsFp); return(statsFp); }
public async Task TestGcReadFailure() { string cacheConfig = TestType.NewCache(nameof(TestGcReadFailure), true); BasicFilesystemCache cache = (await InitializeCacheAsync(cacheConfig).SuccessAsync()) as BasicFilesystemCache; XAssert.IsNotNull(cache, "Failed to create cache for GC tests!"); PipDefinition[] pips = { new PipDefinition("Pip1", pipSize: 3), new PipDefinition("Pip2", pipSize: 4) }; bool disconnectErrorReported = false; // Assumes that the cache will be erroring out because of a file access error and not because of some // other random reason. cache.SuscribeForCacheStateDegredationFailures((failure) => { disconnectErrorReported = true; }); CacheEntries files = await BuildPipsAndGetCacheEntries(cache, "Build", pips); // We will hold on to one of the files in the fingerprints in such a way as to fail the GC FileInfo fileInfo = files.FingerprintFiles.Values.Last(); using (var fileStream = fileInfo.Open(FileMode.Open, FileAccess.Read, FileShare.None)) { var failureExpected = cache.CollectUnreferencedCasItems(m_output); XAssert.IsFalse(failureExpected.Succeeded, "Failed to stop CAS GC even when a fingerprint was not readable!"); } // This time, we need to hold onto a session file - preventing the reading of roots var sessionFiles = new DirectoryInfo(cache.SessionRoot).GetFiles(); using (var fileStream = sessionFiles[0].Open(FileMode.Open, FileAccess.Read, FileShare.None)) { var failureExpected = cache.CollectUnreferencedFingerprints(m_output); XAssert.IsFalse(failureExpected.Succeeded, "Failed to stop Fingerprint GC even when a session was not readable!"); } // After these errors, the cache should have disconnected due to lack of access to the session file XAssert.IsTrue(cache.IsDisconnected); XAssert.IsTrue(disconnectErrorReported); AssertSuccess(await cache.ShutdownAsync()); }
private void ParallelConcurrentFullGc(BasicFilesystemCache cache, string action) { // I picked 16 concurrent as a way to really push the the tests // Making this higher significantly increases the test time without // actually increasing test coverage. Even at 16 this is relatively // high contention rates (which requires the skipped/retry flag). var gcTasks = new Task <Dictionary <string, double> > [16]; for (int i = 0; i < gcTasks.Length; i++) { string name = string.Format("{0}:ParrallelConcurrentFullGc-{1:X}", action, i); gcTasks[i] = Task.Run(() => { Dictionary <string, double> statsFp = cache.CollectUnreferencedFingerprints(m_output).Success("{0}:CollectUnreferencedFingerprints\n{1}", name); Dictionary <string, double> statsCas = cache.CollectUnreferencedCasItems(m_output).Success("{0}:CollectUnreferencedCasItems\n{1}", name); foreach (var kv in statsCas) { statsFp.Add(kv.Key, kv.Value); } return(statsFp); }); } Task.WaitAll(gcTasks); // render the stats in a sane way for the concurrent GCs // Note that the GC is specifically designed to just let // files that are contended stay in place and then be // collected next time the GC runs. The GC will complain // if the cache is in error. for (int i = 0; i < gcTasks.Length; i++) { m_output.WriteLine("Concurrent GC #{0} - {1}", i, action); var stats = gcTasks[i].Result; m_output.LogStats(stats); } }