public async Task RecoveryTest(int expectedNumberOfAttempts, bool isFailureScenario) { var traceWriter = new TestTraceWriter(TraceLevel.Verbose); using (var directory = new TempDirectory()) using (var watcher = new AutoRecoveringFileSystemWatcher(directory.Path, traceWriter: traceWriter)) { Directory.Delete(directory.Path, true); string fileWatcherLogPrefix = $"File watcher: ('{directory.Path}')"; // 1 trace per attempt + 1 trace per failed attempt int expectedTracesBeforeRecovery = (expectedNumberOfAttempts * 2) - 1; // Before + recovery trace int expectedTracesAfterRecovery = expectedTracesBeforeRecovery + 1; await TestHelpers.Await(() => traceWriter.Traces.Count == expectedTracesBeforeRecovery, pollingInterval: 500); if (isFailureScenario) { watcher.Dispose(); } else { Directory.CreateDirectory(directory.Path); } await TestHelpers.Await(() => traceWriter.Traces.Count == expectedTracesAfterRecovery, pollingInterval: 500); TraceEvent failureEvent = traceWriter.Traces.First(); var retryEvents = traceWriter.Traces.Where(t => t.Level == TraceLevel.Warning).Skip(1).ToList(); Assert.Equal(TraceLevel.Warning, failureEvent.Level); Assert.Contains("Failure detected", failureEvent.Message); Assert.Equal(expectedNumberOfAttempts - 1, retryEvents.Count); // Validate that our the events happened with the expected intervals DateTime previoustTimeStamp = failureEvent.Timestamp; for (int i = 0; i < retryEvents.Count; i++) { long expectedInterval = Convert.ToInt64((Math.Pow(2, i + 1) - 1) / 2); TraceEvent currentEvent = retryEvents[i]; var actualInterval = currentEvent.Timestamp - previoustTimeStamp; previoustTimeStamp = currentEvent.Timestamp; Assert.Equal(expectedInterval, (int)actualInterval.TotalSeconds); } Assert.True(traceWriter.Traces.All(t => t.Message.StartsWith(fileWatcherLogPrefix))); if (isFailureScenario) { Assert.Contains("Recovery process aborted.", traceWriter.Traces.Last().Message); } else { Assert.Contains("File watcher recovered.", traceWriter.Traces.Last().Message); } } }