public async Task TestGcCorruptedFingerprint()
{
string cacheConfig = TestType.NewCache(nameof(TestGcCorruptedFingerprint), true);
BasicFilesystemCache cache = (await InitializeCacheAsync(cacheConfig).SuccessAsync()) as BasicFilesystemCache;
XAssert.IsNotNull(cache, "Failed to create cache for GC tests!");
PipDefinition[] pips =
{
new PipDefinition("Pip1"),
new PipDefinition("Pip2"),
new PipDefinition("Pip3")
};
CacheEntries files = await BuildPipsAndGetCacheEntries(cache, "Build", pips);
// We will corrupt one of the fingerprints
var fp = files.FingerprintFiles.Values.ToArray();
// Corrupt the fingerprint by keeping the same size but writing all zeros
// Which is a common corruption on hardware/OS resets.
long length = fp[0].Length;
using (FileStream fs = fp[0].OpenWrite())
{
while (length-- > 0)
{
fs.WriteByte(0);
}
}
// Corrupt the second fingerprint by trimming the file length
using (FileStream fs = fp[1].OpenWrite())
{
fs.SetLength(0);
}
// Now do a GC that reads the fingerprints (collecting the CAS items)
Dictionary <string, double> statsCas = cache.CollectUnreferencedCasItems(m_output).Success("{0}:CollectUnreferencedCasItems - corrupted fingerprint");
m_output.LogStats(statsCas);
// There should now be only 1 valid fingerprint and 2 different
// invalid ones.
XAssert.AreEqual(statsCas["CAS_ReadRoots_Fingerprints"], 1);
XAssert.AreEqual(statsCas["CAS_ReadRoots_InvalidFingerprints"], 2);
// Check that the partition counts match as needed
XAssert.AreEqual((TestType is TestBasicFilesystemSharded) ? TestBasicFilesystemSharded.SHARD_COUNT : 1, statsCas["CAS_Partitions"]);
AssertSuccess(await cache.ShutdownAsync());
}
private Dictionary <string, double> FullGc(BasicFilesystemCache cache, string action = "FullGc")
{
Dictionary <string, double> statsFp = cache.CollectUnreferencedFingerprints(m_output).Success("{0}:CollectUnreferencedFingerprints\n{1}", action);
Dictionary <string, double> statsCas = cache.CollectUnreferencedCasItems(m_output).Success("{0}:CollectUnreferencedCasItems\n{1}", action);
foreach (var kv in statsCas)
{
statsFp.Add(kv.Key, kv.Value);
}
m_output.LogStats(statsFp);
return(statsFp);
}
public async Task TestGcReadFailure()
{
string cacheConfig = TestType.NewCache(nameof(TestGcReadFailure), true);
BasicFilesystemCache cache = (await InitializeCacheAsync(cacheConfig).SuccessAsync()) as BasicFilesystemCache;
XAssert.IsNotNull(cache, "Failed to create cache for GC tests!");
PipDefinition[] pips =
{
new PipDefinition("Pip1", pipSize: 3),
new PipDefinition("Pip2", pipSize: 4)
};
bool disconnectErrorReported = false;
// Assumes that the cache will be erroring out because of a file access error and not because of some
// other random reason.
cache.SuscribeForCacheStateDegredationFailures((failure) => { disconnectErrorReported = true; });
CacheEntries files = await BuildPipsAndGetCacheEntries(cache, "Build", pips);
// We will hold on to one of the files in the fingerprints in such a way as to fail the GC
FileInfo fileInfo = files.FingerprintFiles.Values.Last();
using (var fileStream = fileInfo.Open(FileMode.Open, FileAccess.Read, FileShare.None))
{
var failureExpected = cache.CollectUnreferencedCasItems(m_output);
XAssert.IsFalse(failureExpected.Succeeded, "Failed to stop CAS GC even when a fingerprint was not readable!");
}
// This time, we need to hold onto a session file - preventing the reading of roots
var sessionFiles = new DirectoryInfo(cache.SessionRoot).GetFiles();
using (var fileStream = sessionFiles[0].Open(FileMode.Open, FileAccess.Read, FileShare.None))
{
var failureExpected = cache.CollectUnreferencedFingerprints(m_output);
XAssert.IsFalse(failureExpected.Succeeded, "Failed to stop Fingerprint GC even when a session was not readable!");
}
// After these errors, the cache should have disconnected due to lack of access to the session file
XAssert.IsTrue(cache.IsDisconnected);
XAssert.IsTrue(disconnectErrorReported);
AssertSuccess(await cache.ShutdownAsync());
}
private void ParallelConcurrentFullGc(BasicFilesystemCache cache, string action)
{
// I picked 16 concurrent as a way to really push the the tests
// Making this higher significantly increases the test time without
// actually increasing test coverage. Even at 16 this is relatively
// high contention rates (which requires the skipped/retry flag).
var gcTasks = new Task <Dictionary <string, double> > [16];
for (int i = 0; i < gcTasks.Length; i++)
{
string name = string.Format("{0}:ParrallelConcurrentFullGc-{1:X}", action, i);
gcTasks[i] = Task.Run(() =>
{
Dictionary <string, double> statsFp = cache.CollectUnreferencedFingerprints(m_output).Success("{0}:CollectUnreferencedFingerprints\n{1}", name);
Dictionary <string, double> statsCas = cache.CollectUnreferencedCasItems(m_output).Success("{0}:CollectUnreferencedCasItems\n{1}", name);
foreach (var kv in statsCas)
{
statsFp.Add(kv.Key, kv.Value);
}
return(statsFp);
});
}
Task.WaitAll(gcTasks);
// render the stats in a sane way for the concurrent GCs
// Note that the GC is specifically designed to just let
// files that are contended stay in place and then be
// collected next time the GC runs. The GC will complain
// if the cache is in error.
for (int i = 0; i < gcTasks.Length; i++)
{
m_output.WriteLine("Concurrent GC #{0} - {1}", i, action);
var stats = gcTasks[i].Result;
m_output.LogStats(stats);
}
}
