public async Task RecoveryTest(int expectedNumberOfAttempts, bool isFailureScenario)
{
var traceWriter = new TestTraceWriter(TraceLevel.Verbose);
using (var directory = new TempDirectory())
using (var watcher = new AutoRecoveringFileSystemWatcher(directory.Path, traceWriter: traceWriter))
{
Directory.Delete(directory.Path, true);
string fileWatcherLogPrefix = $"File watcher: ('{directory.Path}')";
// 1 trace per attempt + 1 trace per failed attempt
int expectedTracesBeforeRecovery = (expectedNumberOfAttempts * 2) - 1;
// Before + recovery trace
int expectedTracesAfterRecovery = expectedTracesBeforeRecovery + 1;
await TestHelpers.Await(() => traceWriter.Traces.Count == expectedTracesBeforeRecovery, pollingInterval: 500);
if (isFailureScenario)
{
watcher.Dispose();
}
else
{
Directory.CreateDirectory(directory.Path);
}
await TestHelpers.Await(() => traceWriter.Traces.Count == expectedTracesAfterRecovery, pollingInterval: 500);
TraceEvent failureEvent = traceWriter.Traces.First();
var retryEvents = traceWriter.Traces.Where(t => t.Level == TraceLevel.Warning).Skip(1).ToList();
Assert.Equal(TraceLevel.Warning, failureEvent.Level);
Assert.Contains("Failure detected", failureEvent.Message);
Assert.Equal(expectedNumberOfAttempts - 1, retryEvents.Count);
// Validate that our the events happened with the expected intervals
DateTime previoustTimeStamp = failureEvent.Timestamp;
for (int i = 0; i < retryEvents.Count; i++)
{
long expectedInterval = Convert.ToInt64((Math.Pow(2, i + 1) - 1) / 2);
TraceEvent currentEvent = retryEvents[i];
var actualInterval = currentEvent.Timestamp - previoustTimeStamp;
previoustTimeStamp = currentEvent.Timestamp;
Assert.Equal(expectedInterval, (int)actualInterval.TotalSeconds);
}
Assert.True(traceWriter.Traces.All(t => t.Message.StartsWith(fileWatcherLogPrefix)));
if (isFailureScenario)
{
Assert.Contains("Recovery process aborted.", traceWriter.Traces.Last().Message);
}
else
{
Assert.Contains("File watcher recovered.", traceWriter.Traces.Last().Message);
}
}
}
