private static void ReplayTest()
{
TelemetryClient = new CoyoteTelemetryClient(Configuration);
TelemetryClient.TrackEventAsync("replay").Wait();
if (Debugger.IsAttached)
{
TelemetryClient.TrackEventAsync("replay-debug").Wait();
}
Stopwatch watch = new Stopwatch();
watch.Start();
// Set some replay specific options.
Configuration.SchedulingStrategy = "replay";
Configuration.EnableColoredConsoleOutput = true;
Configuration.DisableEnvironmentExit = false;
// Load the configuration of the assembly to be replayed.
LoadAssemblyConfiguration(Configuration.AssemblyToBeAnalyzed);
Console.WriteLine($". Replaying {Configuration.ScheduleFile}");
TestingEngine engine = TestingEngine.Create(Configuration);
engine.Run();
Console.WriteLine(engine.GetReport());
watch.Stop();
if (!Debugger.IsAttached)
{
TelemetryClient.TrackMetricAsync("replay-time", watch.Elapsed.TotalSeconds).Wait();
}
}
/// <summary>
/// Initializes a new instance of the <see cref="TestingEngine"/> class.
/// </summary>
private TestingEngine(Configuration configuration, TestMethodInfo testMethodInfo)
{
this.Configuration = configuration;
this.TestMethodInfo = testMethodInfo;
this.DefaultLogger = new ConsoleLogger()
{
LogLevel = configuration.LogLevel
};
this.Logger = this.DefaultLogger;
this.Profiler = new Profiler();
this.PerIterationCallbacks = new HashSet <Action <uint> >();
this.TestReport = new TestReport(configuration);
this.ReadableTrace = string.Empty;
this.ReproducibleTrace = string.Empty;
this.CancellationTokenSource = new CancellationTokenSource();
this.PrintGuard = 1;
if (configuration.IsDebugVerbosityEnabled)
{
IO.Debug.IsEnabled = true;
}
// Do some sanity checking.
string error = string.Empty;
if (configuration.IsConcurrencyFuzzingEnabled &&
(configuration.SchedulingStrategy is "replay" || configuration.ScheduleFile.Length > 0))
{
error = "Replaying a bug trace is not supported in concurrency fuzzing.";
}
if (configuration.SchedulingStrategy is "portfolio")
{
error = "Portfolio testing strategy is only available in parallel testing.";
}
if (!string.IsNullOrEmpty(error))
{
if (configuration.DisableEnvironmentExit)
{
throw new Exception(error);
}
else
{
Error.ReportAndExit(error);
}
}
this.Scheduler = OperationScheduler.Setup(configuration);
if (TelemetryClient is null)
{
TelemetryClient = new CoyoteTelemetryClient(this.Configuration);
}
}
/// <summary>
/// Runs the Coyote testing scheduler.
/// </summary>
internal void Run()
{
Console.WriteLine($"... Started the testing task scheduler (process:{Process.GetCurrentProcess().Id}).");
// Start the local server.
this.StartServer();
this.Profiler.StartMeasuringExecutionTime();
if (this.IsRunOutOfProcess)
{
using (var telemetryClient = new CoyoteTelemetryClient(this.Configuration))
{
telemetryClient.TrackEventAsync("test").Wait();
Stopwatch watch = new Stopwatch();
watch.Start();
this.CreateParallelTestingProcesses();
if (this.Configuration.WaitForTestingProcesses)
{
this.WaitForParallelTestingProcesses().Wait();
}
else
{
this.RunParallelTestingProcesses();
}
watch.Stop();
if (this.GlobalTestReport.NumOfFoundBugs > 0)
{
telemetryClient.TrackMetricAsync("test-bugs", this.GlobalTestReport.NumOfFoundBugs).Wait();
}
if (!Debugger.IsAttached)
{
telemetryClient.TrackMetricAsync("test-time", watch.Elapsed.TotalSeconds).Wait();
}
}
}
else
{
this.CreateAndRunInMemoryTestingProcess();
}
this.Profiler.StopMeasuringExecutionTime();
// Stop listening and close the server.
this.StopServer();
if (!IsProcessCanceled)
{
// Merges and emits the test report.
this.EmitTestReport();
}
}
/// <summary>
/// Initializes a new instance of the <see cref="TestingEngine"/> class.
/// </summary>
private TestingEngine(Configuration configuration, TestMethodInfo testMethodInfo)
{
this.Configuration = configuration;
this.TestMethodInfo = testMethodInfo;
this.DefaultLogger = new ConsoleLogger()
{
LogLevel = configuration.LogLevel
};
this.Logger = this.DefaultLogger;
this.Profiler = new Profiler();
this.PerIterationCallbacks = new HashSet <Action <uint> >();
this.TestReport = new TestReport(configuration);
this.ReadableTrace = string.Empty;
this.ReproducibleTrace = string.Empty;
this.CancellationTokenSource = new CancellationTokenSource();
this.PrintGuard = 1;
if (configuration.IsDebugVerbosityEnabled)
{
IO.Debug.IsEnabled = true;
}
if (configuration.SchedulingStrategy is "portfolio")
{
var msg = "Portfolio testing strategy is only " +
"available in parallel testing.";
if (configuration.DisableEnvironmentExit)
{
throw new Exception(msg);
}
else
{
Error.ReportAndExit(msg);
}
}
this.SchedulingContext = SchedulingContext.Setup(configuration, this.Logger);
if (TelemetryClient is null)
{
TelemetryClient = new CoyoteTelemetryClient(this.Configuration);
}
}
private static void Main(string[] args)
{
// Save these so we can force output to happen even if TestingProcess has re-routed it.
StdOut = Console.Out;
StdError = Console.Error;
AppDomain.CurrentDomain.ProcessExit += OnProcessExit;
AppDomain.CurrentDomain.UnhandledException += OnUnhandledException;
Console.CancelKeyPress += OnProcessCanceled;
// Parses the command line options to get the configuration.
Configuration = Configuration.Create();
Configuration.TelemetryServerPath = typeof(Program).Assembly.Location;
var result = CoyoteTelemetryClient.GetOrCreateMachineId().Result;
bool firstTime = result.Item2;
var options = new CommandLineOptions();
if (!options.Parse(args, Configuration))
{
options.PrintHelp(Console.Out);
if (!firstTime && Configuration.EnableTelemetry)
{
CoyoteTelemetryClient.PrintTelemetryMessage(Console.Out);
}
Environment.Exit(1);
}
Configuration.PlatformVersion = GetPlatformVersion();
if (!Configuration.RunAsParallelBugFindingTask)
{
if (firstTime)
{
string version = typeof(Microsoft.Coyote.Runtime.CoyoteRuntime).Assembly.GetName().Version.ToString();
Console.WriteLine("Welcome to Microsoft Coyote {0}", version);
Console.WriteLine("----------------------------{0}", new string('-', version.Length));
if (Configuration.EnableTelemetry)
{
CoyoteTelemetryClient.PrintTelemetryMessage(Console.Out);
}
TelemetryClient = new CoyoteTelemetryClient(Configuration);
TelemetryClient.TrackEventAsync("welcome").Wait();
}
Console.WriteLine("Microsoft (R) Coyote version {0} for .NET{1}",
typeof(CommandLineOptions).Assembly.GetName().Version,
GetDotNetVersion());
Console.WriteLine("Copyright (C) Microsoft Corporation. All rights reserved.");
Console.WriteLine();
}
SetEnvironment(Configuration);
switch (Configuration.ToolCommand.ToLower())
{
case "test":
RunTest();
break;
case "replay":
ReplayTest();
break;
case "rewrite":
RewriteAssemblies();
break;
case "telemetry":
RunServer();
break;
}
}
/// <summary>
/// Initializes a new instance of the <see cref="TestingEngine"/> class.
/// </summary>
private TestingEngine(Configuration configuration, TestMethodInfo testMethodInfo)
{
this.Configuration = configuration;
this.TestMethodInfo = testMethodInfo;
this.DefaultLogger = new ConsoleLogger()
{
LogLevel = configuration.LogLevel
};
this.Logger = this.DefaultLogger;
this.Profiler = new Profiler();
this.PerIterationCallbacks = new HashSet <Action <uint> >();
// Initializes scheduling strategy specific components.
this.RandomValueGenerator = new RandomValueGenerator(configuration);
this.TestReport = new TestReport(configuration);
this.ReadableTrace = string.Empty;
this.ReproducibleTrace = string.Empty;
this.CancellationTokenSource = new CancellationTokenSource();
this.PrintGuard = 1;
if (configuration.IsDebugVerbosityEnabled)
{
IO.Debug.IsEnabled = true;
}
if (!configuration.UserExplicitlySetLivenessTemperatureThreshold &&
configuration.MaxFairSchedulingSteps > 0)
{
configuration.LivenessTemperatureThreshold = configuration.MaxFairSchedulingSteps / 2;
}
if (configuration.SchedulingStrategy is "replay")
{
var scheduleDump = this.GetScheduleForReplay(out bool isFair);
ScheduleTrace schedule = new ScheduleTrace(scheduleDump);
this.Strategy = new ReplayStrategy(configuration, schedule, isFair);
}
else if (configuration.SchedulingStrategy is "interactive")
{
configuration.TestingIterations = 1;
configuration.PerformFullExploration = false;
configuration.IsVerbose = true;
this.Strategy = new InteractiveStrategy(configuration, this.Logger);
}
else if (configuration.SchedulingStrategy is "random")
{
this.Strategy = new RandomStrategy(configuration.MaxFairSchedulingSteps, this.RandomValueGenerator);
}
else if (configuration.SchedulingStrategy is "pct")
{
this.Strategy = new PCTStrategy(configuration.MaxUnfairSchedulingSteps, configuration.StrategyBound,
this.RandomValueGenerator);
}
else if (configuration.SchedulingStrategy is "fairpct")
{
var prefixLength = configuration.SafetyPrefixBound is 0 ?
configuration.MaxUnfairSchedulingSteps : configuration.SafetyPrefixBound;
var prefixStrategy = new PCTStrategy(prefixLength, configuration.StrategyBound, this.RandomValueGenerator);
var suffixStrategy = new RandomStrategy(configuration.MaxFairSchedulingSteps, this.RandomValueGenerator);
this.Strategy = new ComboStrategy(prefixStrategy, suffixStrategy);
}
else if (configuration.SchedulingStrategy is "probabilistic")
{
this.Strategy = new ProbabilisticRandomStrategy(configuration.MaxFairSchedulingSteps,
configuration.StrategyBound, this.RandomValueGenerator);
}
else if (configuration.SchedulingStrategy is "dfs")
{
this.Strategy = new DFSStrategy(configuration.MaxUnfairSchedulingSteps);
}
else if (configuration.SchedulingStrategy is "rl")
{
this.Strategy = new QLearningStrategy(configuration.AbstractionLevel, configuration.MaxUnfairSchedulingSteps, this.RandomValueGenerator);
}
else if (configuration.SchedulingStrategy is "portfolio")
{
var msg = "Portfolio testing strategy is only " +
"available in parallel testing.";
if (configuration.DisableEnvironmentExit)
{
throw new Exception(msg);
}
else
{
Error.ReportAndExit(msg);
}
}
if (configuration.SchedulingStrategy != "replay" &&
configuration.ScheduleFile.Length > 0)
{
var scheduleDump = this.GetScheduleForReplay(out bool isFair);
ScheduleTrace schedule = new ScheduleTrace(scheduleDump);
this.Strategy = new ReplayStrategy(configuration, schedule, isFair, this.Strategy);
}
if (TelemetryClient is null)
{
TelemetryClient = new CoyoteTelemetryClient(this.Configuration);
}
}
private static void RunTest()
{
if (Configuration.RunAsParallelBugFindingTask)
{
// This is being run as the child test process.
if (Configuration.ParallelDebug)
{
Console.WriteLine("Attach Debugger and press ENTER to continue...");
Console.ReadLine();
}
// Load the configuration of the assembly to be tested.
LoadAssemblyConfiguration(Configuration.AssemblyToBeAnalyzed);
TestingProcess testingProcess = TestingProcess.Create(Configuration);
testingProcess.Run();
return;
}
TelemetryClient = new CoyoteTelemetryClient(Configuration);
TelemetryClient.TrackEventAsync("test").Wait();
if (Debugger.IsAttached)
{
TelemetryClient.TrackEventAsync("test-debug").Wait();
}
Stopwatch watch = new Stopwatch();
watch.Start();
if (Configuration.ReportCodeCoverage || Configuration.ReportActivityCoverage)
{
// This has to be here because both forms of coverage require it.
CodeCoverageInstrumentation.SetOutputDirectory(Configuration, makeHistory: true);
}
if (Configuration.ReportCodeCoverage)
{
// Instruments the program under test for code coverage.
CodeCoverageInstrumentation.Instrument(Configuration);
// Starts monitoring for code coverage.
CodeCoverageMonitor.Start(Configuration);
}
Console.WriteLine(". Testing " + Configuration.AssemblyToBeAnalyzed);
if (!string.IsNullOrEmpty(Configuration.TestMethodName))
{
Console.WriteLine("... Method {0}", Configuration.TestMethodName);
}
// Creates and runs the testing process scheduler.
int bugs = TestingProcessScheduler.Create(Configuration).Run();
if (bugs > 0)
{
TelemetryClient.TrackMetricAsync("test-bugs", bugs).Wait();
}
Console.WriteLine(". Done");
watch.Stop();
if (!Debugger.IsAttached)
{
TelemetryClient.TrackMetricAsync("test-time", watch.Elapsed.TotalSeconds).Wait();
}
}
