private CorruptedMemosDbRecovery MarkFailureAndTryRecover()
{
BuildXLContext context;
IConfiguration configuration;
CreateContextAndConfiguration(out context, out configuration);
WriteFakeMemosDb(context, configuration);
FailureRecoveryAggregator recovery = FailureRecoveryFactory.Create(LoggingContext, context.PathTable, configuration);
bool markFailure = recovery.TryMarkFailure(new BuildXLException(string.Empty), ExceptionRootCause.CorruptedCache);
XAssert.IsTrue(markFailure);
var corruptedMemosDbRecovery = new CorruptedMemosDbRecovery(context.PathTable, configuration);
XAssert.IsTrue(corruptedMemosDbRecovery.ShouldRecover());
CreateContextAndConfiguration(out context, out configuration);
recovery = FailureRecoveryFactory.Create(LoggingContext, context.PathTable, configuration);
bool tryRecover = recovery.TryRecoverIfNeeded();
XAssert.IsTrue(tryRecover);
corruptedMemosDbRecovery = new CorruptedMemosDbRecovery(context.PathTable, configuration);
XAssert.IsFalse(File.Exists(GetMemosDbPath(context, configuration)));
return(corruptedMemosDbRecovery);
}
public void TestCatastrophicFailureRecovery()
{
// Run some valid module
SetupTestData();
RunEngine();
// List the files in the engine cache after a valid run
var engineCacheDirectory = Configuration.Layout.EngineCacheDirectory.ToString(Context.PathTable);
var engineCacheFilesList = new List <string>();
FileUtilities.EnumerateDirectoryEntries(engineCacheDirectory, (file, attributes) =>
{
if (!attributes.HasFlag(FileAttributes.Directory))
{
engineCacheFilesList.Add(file);
}
});
var recovery = FailureRecoveryFactory.Create(LoggingContext, Context.PathTable, Configuration);
// This will trigger the recovery mechanism for unknown catastrophic errors, which is to log and remove the engine state (EngineCache folder)
XAssert.IsTrue(recovery.TryMarkFailure(new BuildXLException("fake failure"), ExceptionRootCause.Unknown));
// List the files in the logs directory for corrupt engine cache files
var logsDirectory = Configuration.Logging.EngineCacheCorruptFilesLogDirectory.ToString(Context.PathTable);
var logsFilesList = new HashSet <string>();
FileUtilities.EnumerateDirectoryEntries(logsDirectory, (file, attributes) =>
{
logsFilesList.Add(file);
});
var childrenCount = Directory.GetFiles(engineCacheDirectory, "*", SearchOption.TopDirectoryOnly).Length;
var expectedCount = -1;
// File content table has a special exclusion from the removal policy for performance reasons, but it should still be copied to logs
// (Unless the file content table doesn't exist in the engine cache, then it doesn't need to exist in the logs)
var engineCacheFileContentTablePath = Configuration.Layout.FileContentTableFile.ToString(Context.PathTable);
var fileContentTableFile = Path.GetFileName(engineCacheFileContentTablePath);
// Make sure file content table was copied to logs
XAssert.Contains(logsFilesList, fileContentTableFile);
expectedCount = 1;
// Make sure file content table file exists in the engine cache directory after recovery
XAssert.IsTrue(File.Exists(engineCacheFileContentTablePath));
// Check to make sure the engine cache directory is empty except for maybe the file content table
XAssert.AreEqual(expectedCount, childrenCount);
// Check to make sure all the file from the engine cache directory ended up in the logs directory
foreach (var file in engineCacheFilesList)
{
XAssert.Contains(logsFilesList, file);
}
}