/// <summary>
/// Initializes a new instance of the <see cref="TestingEngine"/> class.
/// </summary>
private TestingEngine(Configuration configuration, TestMethodInfo testMethodInfo)
{
this.Configuration = configuration;
this.TestMethodInfo = testMethodInfo;
this.DefaultLogger = new ConsoleLogger()
{
LogLevel = configuration.LogLevel
};
this.Logger = this.DefaultLogger;
this.Profiler = new Profiler();
this.PerIterationCallbacks = new HashSet <Action <uint> >();
this.TestReport = new TestReport(configuration);
this.ReadableTrace = string.Empty;
this.ReproducibleTrace = string.Empty;
this.CancellationTokenSource = new CancellationTokenSource();
this.PrintGuard = 1;
if (configuration.IsDebugVerbosityEnabled)
{
IO.Debug.IsEnabled = true;
}
// Do some sanity checking.
string error = string.Empty;
if (configuration.IsConcurrencyFuzzingEnabled &&
(configuration.SchedulingStrategy is "replay" || configuration.ScheduleFile.Length > 0))
{
error = "Replaying a bug trace is not supported in concurrency fuzzing.";
}
if (configuration.SchedulingStrategy is "portfolio")
{
error = "Portfolio testing strategy is only available in parallel testing.";
}
if (!string.IsNullOrEmpty(error))
{
if (configuration.DisableEnvironmentExit)
{
throw new Exception(error);
}
else
{
Error.ReportAndExit(error);
}
}
this.Scheduler = OperationScheduler.Setup(configuration);
if (TelemetryClient is null)
{
TelemetryClient = new CoyoteTelemetryClient(this.Configuration);
}
}
/// <summary>
/// Creates a new systematic testing engine.
/// </summary>
public static TestingEngine Create(Configuration configuration)
{
try
{
TestMethodInfo testMethodInfo = TestMethodInfo.Create(configuration);
return(new TestingEngine(configuration, testMethodInfo));
}
catch (Exception ex)
{
Error.Report(ex.Message);
throw;
}
}
/// <summary>
/// Creates a new systematic testing engine.
/// </summary>
public static TestingEngine Create(Configuration configuration, Assembly assembly)
{
TestMethodInfo testMethodInfo = null;
try
{
testMethodInfo = TestMethodInfo.GetFromAssembly(assembly, configuration.TestMethodName);
}
catch
{
Error.ReportAndExit($"Failed to get test method '{configuration.TestMethodName}' from assembly '{assembly.FullName}'");
}
return(new TestingEngine(configuration, testMethodInfo));
}
/// <summary>
/// Creates a new systematic testing engine.
/// </summary>
public static TestingEngine Create(Configuration configuration)
{
TestMethodInfo testMethodInfo = null;
try
{
testMethodInfo = TestMethodInfo.Create(configuration);
}
catch (Exception ex)
{
Error.ReportAndExit(ex.Message);
}
return(new TestingEngine(configuration, testMethodInfo));
}
/// <summary>
/// Initializes a new instance of the <see cref="TestingEngine"/> class.
/// </summary>
private TestingEngine(Configuration configuration, TestMethodInfo testMethodInfo)
{
this.Configuration = configuration;
this.TestMethodInfo = testMethodInfo;
this.DefaultLogger = new ConsoleLogger()
{
LogLevel = configuration.LogLevel
};
this.Logger = this.DefaultLogger;
this.Profiler = new Profiler();
this.PerIterationCallbacks = new HashSet <Action <uint> >();
this.TestReport = new TestReport(configuration);
this.ReadableTrace = string.Empty;
this.ReproducibleTrace = string.Empty;
this.CancellationTokenSource = new CancellationTokenSource();
this.PrintGuard = 1;
if (configuration.IsDebugVerbosityEnabled)
{
IO.Debug.IsEnabled = true;
}
if (configuration.SchedulingStrategy is "portfolio")
{
var msg = "Portfolio testing strategy is only " +
"available in parallel testing.";
if (configuration.DisableEnvironmentExit)
{
throw new Exception(msg);
}
else
{
Error.ReportAndExit(msg);
}
}
this.SchedulingContext = SchedulingContext.Setup(configuration, this.Logger);
if (TelemetryClient is null)
{
TelemetryClient = new CoyoteTelemetryClient(this.Configuration);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="TestingEngine"/> class.
/// </summary>
private TestingEngine(Configuration configuration, TestMethodInfo testMethodInfo)
{
this.Configuration = configuration;
this.TestMethodInfo = testMethodInfo;
this.DefaultLogger = new ConsoleLogger()
{
LogLevel = configuration.LogLevel
};
this.Logger = this.DefaultLogger;
this.Profiler = new Profiler();
this.PerIterationCallbacks = new HashSet <Action <uint> >();
this.TestReport = new TestReport(configuration);
this.ReadableTrace = string.Empty;
this.ReproducibleTrace = string.Empty;
this.CancellationTokenSource = new CancellationTokenSource();
this.PrintGuard = 1;
if (configuration.IsDebugVerbosityEnabled)
{
IO.Debug.IsEnabled = true;
}
// Do some sanity checking.
string error = string.Empty;
if (configuration.IsSystematicFuzzingEnabled &&
(configuration.SchedulingStrategy is "replay" || configuration.ScheduleFile.Length > 0))
{
error = "Replaying a bug trace is not currently supported in systematic fuzzing.";
}
if (!string.IsNullOrEmpty(error))
{
Error.Report(error);
throw new InvalidOperationException(error);
}
this.Scheduler = OperationScheduler.Setup(configuration);
TelemetryClient = TelemetryClient.GetOrCreate(this.Configuration);
}
/// <summary>
/// Creates a new systematic testing engine.
/// </summary>
public static TestingEngine Create(Configuration configuration)
{
TestMethodInfo testMethodInfo = null;
try
{
testMethodInfo = TestMethodInfo.Create(configuration);
}
catch (Exception ex)
{
if (configuration.DisableEnvironmentExit)
{
throw;
}
else
{
Error.ReportAndExit(ex.Message);
}
}
return(new TestingEngine(configuration, testMethodInfo));
}
/// <summary>
/// Initializes a new instance of the <see cref="TestingEngine"/> class.
/// </summary>
private TestingEngine(Configuration configuration, TestMethodInfo testMethodInfo)
{
this.Configuration = configuration;
this.TestMethodInfo = testMethodInfo;
this.Logger = new ConsoleLogger();
this.ErrorReporter = new ErrorReporter(configuration, this.Logger);
this.Profiler = new Profiler();
this.PerIterationCallbacks = new HashSet <Action <int> >();
// Initializes scheduling strategy specific components.
this.RandomValueGenerator = new RandomValueGenerator(configuration);
this.TestReport = new TestReport(configuration);
this.ReadableTrace = string.Empty;
this.ReproducableTrace = string.Empty;
this.CancellationTokenSource = new CancellationTokenSource();
this.PrintGuard = 1;
if (!configuration.UserExplicitlySetLivenessTemperatureThreshold &&
configuration.MaxFairSchedulingSteps > 0)
{
configuration.LivenessTemperatureThreshold = configuration.MaxFairSchedulingSteps / 2;
}
if (configuration.SchedulingStrategy is "replay")
{
var scheduleDump = this.GetScheduleForReplay(out bool isFair);
ScheduleTrace schedule = new ScheduleTrace(scheduleDump);
this.Strategy = new ReplayStrategy(configuration, schedule, isFair);
}
else if (configuration.SchedulingStrategy is "interactive")
{
configuration.TestingIterations = 1;
configuration.PerformFullExploration = false;
configuration.IsVerbose = true;
this.Strategy = new InteractiveStrategy(configuration, this.Logger);
}
else if (configuration.SchedulingStrategy is "random")
{
this.Strategy = new RandomStrategy(configuration.MaxFairSchedulingSteps, this.RandomValueGenerator);
}
else if (configuration.SchedulingStrategy is "pct")
{
this.Strategy = new PCTStrategy(configuration.MaxUnfairSchedulingSteps, configuration.StrategyBound,
this.RandomValueGenerator);
}
else if (configuration.SchedulingStrategy is "fairpct")
{
var prefixLength = configuration.SafetyPrefixBound == 0 ?
configuration.MaxUnfairSchedulingSteps : configuration.SafetyPrefixBound;
var prefixStrategy = new PCTStrategy(prefixLength, configuration.StrategyBound, this.RandomValueGenerator);
var suffixStrategy = new RandomStrategy(configuration.MaxFairSchedulingSteps, this.RandomValueGenerator);
this.Strategy = new ComboStrategy(prefixStrategy, suffixStrategy);
}
else if (configuration.SchedulingStrategy is "probabilistic")
{
this.Strategy = new ProbabilisticRandomStrategy(configuration.MaxFairSchedulingSteps,
configuration.StrategyBound, this.RandomValueGenerator);
}
else if (configuration.SchedulingStrategy is "dfs")
{
this.Strategy = new DFSStrategy(configuration.MaxUnfairSchedulingSteps);
}
else if (configuration.SchedulingStrategy is "portfolio")
{
Error.ReportAndExit("Portfolio testing strategy is only " +
"available in parallel testing.");
}
if (configuration.SchedulingStrategy != "replay" &&
configuration.ScheduleFile.Length > 0)
{
var scheduleDump = this.GetScheduleForReplay(out bool isFair);
ScheduleTrace schedule = new ScheduleTrace(scheduleDump);
this.Strategy = new ReplayStrategy(configuration, schedule, isFair, this.Strategy);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="TestingEngine"/> class.
/// </summary>
private TestingEngine(Configuration configuration, TestMethodInfo testMethodInfo)
{
this.Configuration = configuration;
this.TestMethodInfo = testMethodInfo;
this.DefaultLogger = new ConsoleLogger()
{
LogLevel = configuration.LogLevel
};
this.Logger = this.DefaultLogger;
this.Profiler = new Profiler();
this.PerIterationCallbacks = new HashSet <Action <uint> >();
// Initializes scheduling strategy specific components.
this.RandomValueGenerator = new RandomValueGenerator(configuration);
this.TestReport = new TestReport(configuration);
this.ReadableTrace = string.Empty;
this.ReproducibleTrace = string.Empty;
this.CancellationTokenSource = new CancellationTokenSource();
this.PrintGuard = 1;
if (configuration.IsDebugVerbosityEnabled)
{
IO.Debug.IsEnabled = true;
}
if (!configuration.UserExplicitlySetLivenessTemperatureThreshold &&
configuration.MaxFairSchedulingSteps > 0)
{
configuration.LivenessTemperatureThreshold = configuration.MaxFairSchedulingSteps / 2;
}
if (configuration.SchedulingStrategy is "replay")
{
var scheduleDump = this.GetScheduleForReplay(out bool isFair);
ScheduleTrace schedule = new ScheduleTrace(scheduleDump);
this.Strategy = new ReplayStrategy(configuration, schedule, isFair);
}
else if (configuration.SchedulingStrategy is "interactive")
{
configuration.TestingIterations = 1;
configuration.PerformFullExploration = false;
configuration.IsVerbose = true;
this.Strategy = new InteractiveStrategy(configuration, this.Logger);
}
else if (configuration.SchedulingStrategy is "random")
{
this.Strategy = new RandomStrategy(configuration.MaxFairSchedulingSteps, this.RandomValueGenerator);
}
else if (configuration.SchedulingStrategy is "pct")
{
this.Strategy = new PCTStrategy(configuration.MaxUnfairSchedulingSteps, configuration.StrategyBound,
this.RandomValueGenerator);
}
else if (configuration.SchedulingStrategy is "fairpct")
{
var prefixLength = configuration.SafetyPrefixBound is 0 ?
configuration.MaxUnfairSchedulingSteps : configuration.SafetyPrefixBound;
var prefixStrategy = new PCTStrategy(prefixLength, configuration.StrategyBound, this.RandomValueGenerator);
var suffixStrategy = new RandomStrategy(configuration.MaxFairSchedulingSteps, this.RandomValueGenerator);
this.Strategy = new ComboStrategy(prefixStrategy, suffixStrategy);
}
else if (configuration.SchedulingStrategy is "probabilistic")
{
this.Strategy = new ProbabilisticRandomStrategy(configuration.MaxFairSchedulingSteps,
configuration.StrategyBound, this.RandomValueGenerator);
}
else if (configuration.SchedulingStrategy is "dfs")
{
this.Strategy = new DFSStrategy(configuration.MaxUnfairSchedulingSteps);
}
else if (configuration.SchedulingStrategy is "rl")
{
this.Strategy = new QLearningStrategy(configuration.AbstractionLevel, configuration.MaxUnfairSchedulingSteps, this.RandomValueGenerator);
}
else if (configuration.SchedulingStrategy is "portfolio")
{
var msg = "Portfolio testing strategy is only " +
"available in parallel testing.";
if (configuration.DisableEnvironmentExit)
{
throw new Exception(msg);
}
else
{
Error.ReportAndExit(msg);
}
}
if (configuration.SchedulingStrategy != "replay" &&
configuration.ScheduleFile.Length > 0)
{
var scheduleDump = this.GetScheduleForReplay(out bool isFair);
ScheduleTrace schedule = new ScheduleTrace(scheduleDump);
this.Strategy = new ReplayStrategy(configuration, schedule, isFair, this.Strategy);
}
if (TelemetryClient is null)
{
TelemetryClient = new CoyoteTelemetryClient(this.Configuration);
}
}
/// <summary>
/// Runs the next testing iteration for the specified test method.
/// </summary>
private bool RunNextIteration(TestMethodInfo methodInfo, uint iteration)
{
if (!this.Scheduler.InitializeNextIteration(iteration))
{
// The next iteration cannot run, so stop exploring.
return(false);
}
if (!this.Scheduler.IsReplayingSchedule && this.ShouldPrintIteration(iteration + 1))
{
this.Logger.WriteLine(LogSeverity.Important, $"..... Iteration #{iteration + 1}");
// Flush when logging to console.
if (this.Logger is ConsoleLogger)
{
Console.Out.Flush();
}
}
// Runtime used to serialize and test the program in this iteration.
CoyoteRuntime runtime = null;
// Logger used to intercept the program output if no custom logger
// is installed and if verbosity is turned off.
InMemoryLogger runtimeLogger = null;
// Gets a handle to the standard output and error streams.
var stdOut = Console.Out;
var stdErr = Console.Error;
try
{
// Creates a new instance of the controlled runtime.
runtime = new CoyoteRuntime(this.Configuration, this.Scheduler);
// If verbosity is turned off, then intercept the program log, and also redirect
// the standard output and error streams to the runtime logger.
if (!this.Configuration.IsVerbose)
{
runtimeLogger = new InMemoryLogger();
if (this.Logger != this.DefaultLogger)
{
runtimeLogger.UserLogger = this.Logger;
}
runtime.Logger = runtimeLogger;
Console.SetOut(runtimeLogger.TextWriter);
Console.SetError(runtimeLogger.TextWriter);
}
else if (this.Logger != this.DefaultLogger)
{
runtime.Logger = this.Logger;
}
this.InitializeCustomActorLogging(runtime.DefaultActorExecutionContext);
// Runs the test and waits for it to terminate.
Task task = runtime.RunTestAsync(methodInfo.Method, methodInfo.Name);
task.Wait();
// Invokes the user-specified iteration disposal method.
methodInfo.DisposeCurrentIteration();
// Invoke the per iteration callbacks, if any.
foreach (var callback in this.PerIterationCallbacks)
{
callback(iteration);
}
runtime.LogWriter.LogCompletion();
this.GatherTestingStatistics(runtime);
if (!this.Scheduler.IsReplayingSchedule && this.TestReport.NumOfFoundBugs > 0)
{
if (runtimeLogger != null)
{
this.ReadableTrace = string.Empty;
if (this.Configuration.IsTelemetryEnabled)
{
this.ReadableTrace += $"<TelemetryLog> Anonymized telemetry is enabled, see {LearnAboutTelemetryUrl}.\n";
}
this.ReadableTrace += runtimeLogger.ToString();
this.ReadableTrace += this.TestReport.GetText(this.Configuration, "<StrategyLog>");
}
if (runtime.SchedulingPolicy is SchedulingPolicy.Interleaving)
{
this.ReproducibleTrace = this.Scheduler.Trace.Serialize(
this.Configuration, this.Scheduler.IsScheduleFair);
}
}
}
finally
{
if (!this.Configuration.IsVerbose)
{
// Restores the standard output and error streams.
Console.SetOut(stdOut);
Console.SetError(stdErr);
}
if (this.Configuration.IsSystematicFuzzingFallbackEnabled &&
runtime.SchedulingPolicy is SchedulingPolicy.Interleaving &&
(runtime.ExecutionStatus is ExecutionStatus.ConcurrencyUncontrolled ||
runtime.ExecutionStatus is ExecutionStatus.Deadlocked))
{
// Detected uncontrolled concurrency or deadlock, so switch to systematic fuzzing.
this.Scheduler = OperationScheduler.Setup(this.Configuration, SchedulingPolicy.Fuzzing,
this.Scheduler.ValueGenerator);
this.Logger.WriteLine(LogSeverity.Important, $"..... Iteration #{iteration + 1} " +
$"enables systematic fuzzing due to uncontrolled concurrency");
}
else if (runtime.ExecutionStatus is ExecutionStatus.BoundReached)
{
this.Logger.WriteLine(LogSeverity.Important, $"..... Iteration #{iteration + 1} " +
$"hit bound of '{this.Scheduler.StepCount}' scheduling steps");
}
else if (runtime.ExecutionStatus is ExecutionStatus.BugFound)
{
if (!this.Scheduler.IsReplayingSchedule)
{
this.Logger.WriteLine(LogSeverity.Important, $"..... Iteration #{iteration + 1} " +
$"found bug #{this.TestReport.NumOfFoundBugs}");
}
this.Logger.WriteLine(LogSeverity.Error, runtime.BugReport);
}
else if (this.Scheduler.IsReplayingSchedule)
{
this.Logger.WriteLine(LogSeverity.Error, "Failed to reproduce the bug.");
}
// Cleans up the runtime before the next iteration starts.
runtimeLogger?.Close();
runtimeLogger?.Dispose();
runtime?.Dispose();
}
return(true);
}
/// <summary>
/// Creates a new testing task for the specified test method.
/// </summary>
private Task CreateTestingTask(TestMethodInfo methodInfo)
{
return(new Task(() =>
{
this.Logger.WriteLine(LogSeverity.Important, "... Setting up the{0} test:",
string.IsNullOrEmpty(methodInfo.Name) ? string.Empty : $" '{methodInfo.Name}'");
this.Logger.WriteLine(LogSeverity.Important,
$"..... Using the {this.Scheduler.GetDescription()} exploration strategy.");
if (this.Configuration.AttachDebugger)
{
this.Logger.WriteLine(LogSeverity.Important,
$"..... Launching and attaching the debugger.");
Debugger.Launch();
}
try
{
// Invokes the user-specified initialization method.
methodInfo.InitializeAllIterations();
if (this.Scheduler.IsReplayingSchedule)
{
this.Logger.WriteLine(LogSeverity.Important, "... Replaying the trace{0}.",
this.Configuration.ScheduleFile.Length > 0 ? $" from {this.Configuration.ScheduleFile}" : string.Empty);
}
else
{
this.Logger.WriteLine(LogSeverity.Important, "... Running test iterations:");
}
uint iteration = 0;
while (iteration < this.Configuration.TestingIterations || this.Configuration.TestingTimeout > 0)
{
if (this.CancellationTokenSource.IsCancellationRequested)
{
break;
}
// Runs the next iteration.
bool runNext = this.RunNextIteration(methodInfo, iteration);
if ((!this.Configuration.RunTestIterationsToCompletion && this.TestReport.NumOfFoundBugs > 0) ||
this.Scheduler.IsReplayingSchedule || !runNext)
{
break;
}
if (this.Scheduler.ValueGenerator != null && this.Configuration.IsSchedulingSeedIncremental)
{
// Increments the seed in the random number generator (if one is used), to
// capture the seed used by the scheduling strategy in the next iteration.
this.Scheduler.ValueGenerator.Seed += 1;
}
iteration++;
}
// Invokes the user-specified test disposal method.
methodInfo.DisposeAllIterations();
}
catch (Exception ex)
{
Exception innerException = ex;
while (innerException is TargetInvocationException)
{
innerException = innerException.InnerException;
}
if (innerException is AggregateException)
{
innerException = innerException.InnerException;
}
if (!(innerException is TaskCanceledException))
{
ExceptionDispatchInfo.Capture(innerException).Throw();
}
}
}, this.CancellationTokenSource.Token));
}