/// <summary> /// Initializes a new instance of the <see cref="TestingEngine"/> class. /// </summary> private TestingEngine(Configuration configuration, TestMethodInfo testMethodInfo) { this.Configuration = configuration; this.TestMethodInfo = testMethodInfo; this.DefaultLogger = new ConsoleLogger() { LogLevel = configuration.LogLevel }; this.Logger = this.DefaultLogger; this.Profiler = new Profiler(); this.PerIterationCallbacks = new HashSet <Action <uint> >(); this.TestReport = new TestReport(configuration); this.ReadableTrace = string.Empty; this.ReproducibleTrace = string.Empty; this.CancellationTokenSource = new CancellationTokenSource(); this.PrintGuard = 1; if (configuration.IsDebugVerbosityEnabled) { IO.Debug.IsEnabled = true; } // Do some sanity checking. string error = string.Empty; if (configuration.IsConcurrencyFuzzingEnabled && (configuration.SchedulingStrategy is "replay" || configuration.ScheduleFile.Length > 0)) { error = "Replaying a bug trace is not supported in concurrency fuzzing."; } if (configuration.SchedulingStrategy is "portfolio") { error = "Portfolio testing strategy is only available in parallel testing."; } if (!string.IsNullOrEmpty(error)) { if (configuration.DisableEnvironmentExit) { throw new Exception(error); } else { Error.ReportAndExit(error); } } this.Scheduler = OperationScheduler.Setup(configuration); if (TelemetryClient is null) { TelemetryClient = new CoyoteTelemetryClient(this.Configuration); } }
/// <summary> /// Initializes a new instance of the <see cref="TestingEngine"/> class. /// </summary> private TestingEngine(Configuration configuration, TestMethodInfo testMethodInfo) { this.Configuration = configuration; this.TestMethodInfo = testMethodInfo; this.DefaultLogger = new ConsoleLogger() { LogLevel = configuration.LogLevel }; this.Logger = this.DefaultLogger; this.Profiler = new Profiler(); this.PerIterationCallbacks = new HashSet <Action <uint> >(); this.TestReport = new TestReport(configuration); this.ReadableTrace = string.Empty; this.ReproducibleTrace = string.Empty; this.CancellationTokenSource = new CancellationTokenSource(); this.PrintGuard = 1; if (configuration.IsDebugVerbosityEnabled) { IO.Debug.IsEnabled = true; } // Do some sanity checking. string error = string.Empty; if (configuration.IsSystematicFuzzingEnabled && (configuration.SchedulingStrategy is "replay" || configuration.ScheduleFile.Length > 0)) { error = "Replaying a bug trace is not currently supported in systematic fuzzing."; } if (!string.IsNullOrEmpty(error)) { Error.Report(error); throw new InvalidOperationException(error); } this.Scheduler = OperationScheduler.Setup(configuration); TelemetryClient = TelemetryClient.GetOrCreate(this.Configuration); }
/// <summary> /// Runs the next testing iteration for the specified test method. /// </summary> private bool RunNextIteration(TestMethodInfo methodInfo, uint iteration) { if (!this.Scheduler.InitializeNextIteration(iteration)) { // The next iteration cannot run, so stop exploring. return(false); } if (!this.Scheduler.IsReplayingSchedule && this.ShouldPrintIteration(iteration + 1)) { this.Logger.WriteLine(LogSeverity.Important, $"..... Iteration #{iteration + 1}"); // Flush when logging to console. if (this.Logger is ConsoleLogger) { Console.Out.Flush(); } } // Runtime used to serialize and test the program in this iteration. CoyoteRuntime runtime = null; // Logger used to intercept the program output if no custom logger // is installed and if verbosity is turned off. InMemoryLogger runtimeLogger = null; // Gets a handle to the standard output and error streams. var stdOut = Console.Out; var stdErr = Console.Error; try { // Creates a new instance of the controlled runtime. runtime = new CoyoteRuntime(this.Configuration, this.Scheduler); // If verbosity is turned off, then intercept the program log, and also redirect // the standard output and error streams to the runtime logger. if (!this.Configuration.IsVerbose) { runtimeLogger = new InMemoryLogger(); if (this.Logger != this.DefaultLogger) { runtimeLogger.UserLogger = this.Logger; } runtime.Logger = runtimeLogger; Console.SetOut(runtimeLogger.TextWriter); Console.SetError(runtimeLogger.TextWriter); } else if (this.Logger != this.DefaultLogger) { runtime.Logger = this.Logger; } this.InitializeCustomActorLogging(runtime.DefaultActorExecutionContext); // Runs the test and waits for it to terminate. Task task = runtime.RunTestAsync(methodInfo.Method, methodInfo.Name); task.Wait(); // Invokes the user-specified iteration disposal method. methodInfo.DisposeCurrentIteration(); // Invoke the per iteration callbacks, if any. foreach (var callback in this.PerIterationCallbacks) { callback(iteration); } runtime.LogWriter.LogCompletion(); this.GatherTestingStatistics(runtime); if (!this.Scheduler.IsReplayingSchedule && this.TestReport.NumOfFoundBugs > 0) { if (runtimeLogger != null) { this.ReadableTrace = string.Empty; if (this.Configuration.IsTelemetryEnabled) { this.ReadableTrace += $"<TelemetryLog> Anonymized telemetry is enabled, see {LearnAboutTelemetryUrl}.\n"; } this.ReadableTrace += runtimeLogger.ToString(); this.ReadableTrace += this.TestReport.GetText(this.Configuration, "<StrategyLog>"); } if (runtime.SchedulingPolicy is SchedulingPolicy.Interleaving) { this.ReproducibleTrace = this.Scheduler.Trace.Serialize( this.Configuration, this.Scheduler.IsScheduleFair); } } } finally { if (!this.Configuration.IsVerbose) { // Restores the standard output and error streams. Console.SetOut(stdOut); Console.SetError(stdErr); } if (this.Configuration.IsSystematicFuzzingFallbackEnabled && runtime.SchedulingPolicy is SchedulingPolicy.Interleaving && (runtime.ExecutionStatus is ExecutionStatus.ConcurrencyUncontrolled || runtime.ExecutionStatus is ExecutionStatus.Deadlocked)) { // Detected uncontrolled concurrency or deadlock, so switch to systematic fuzzing. this.Scheduler = OperationScheduler.Setup(this.Configuration, SchedulingPolicy.Fuzzing, this.Scheduler.ValueGenerator); this.Logger.WriteLine(LogSeverity.Important, $"..... Iteration #{iteration + 1} " + $"enables systematic fuzzing due to uncontrolled concurrency"); } else if (runtime.ExecutionStatus is ExecutionStatus.BoundReached) { this.Logger.WriteLine(LogSeverity.Important, $"..... Iteration #{iteration + 1} " + $"hit bound of '{this.Scheduler.StepCount}' scheduling steps"); } else if (runtime.ExecutionStatus is ExecutionStatus.BugFound) { if (!this.Scheduler.IsReplayingSchedule) { this.Logger.WriteLine(LogSeverity.Important, $"..... Iteration #{iteration + 1} " + $"found bug #{this.TestReport.NumOfFoundBugs}"); } this.Logger.WriteLine(LogSeverity.Error, runtime.BugReport); } else if (this.Scheduler.IsReplayingSchedule) { this.Logger.WriteLine(LogSeverity.Error, "Failed to reproduce the bug."); } // Cleans up the runtime before the next iteration starts. runtimeLogger?.Close(); runtimeLogger?.Dispose(); runtime?.Dispose(); } return(true); }