/// <summary>
/// Runs the Coyote testing scheduler.
/// </summary>
internal void Run()
{
Console.WriteLine($"... Started the testing task scheduler (process:{Process.GetCurrentProcess().Id}).");
// Start the local server.
this.StartServer();
this.Profiler.StartMeasuringExecutionTime();
if (this.IsRunOutOfProcess)
{
using (var telemetryClient = new CoyoteTelemetryClient(this.Configuration))
{
telemetryClient.TrackEventAsync("test").Wait();
Stopwatch watch = new Stopwatch();
watch.Start();
this.CreateParallelTestingProcesses();
if (this.Configuration.WaitForTestingProcesses)
{
this.WaitForParallelTestingProcesses().Wait();
}
else
{
this.RunParallelTestingProcesses();
}
watch.Stop();
if (this.GlobalTestReport.NumOfFoundBugs > 0)
{
telemetryClient.TrackMetricAsync("test-bugs", this.GlobalTestReport.NumOfFoundBugs).Wait();
}
if (!Debugger.IsAttached)
{
telemetryClient.TrackMetricAsync("test-time", watch.Elapsed.TotalSeconds).Wait();
}
}
}
else
{
this.CreateAndRunInMemoryTestingProcess();
}
this.Profiler.StopMeasuringExecutionTime();
// Stop listening and close the server.
this.StopServer();
if (!IsProcessCanceled)
{
// Merges and emits the test report.
this.EmitTestReport();
}
}
private static void ReplayTest()
{
TelemetryClient = new CoyoteTelemetryClient(Configuration);
TelemetryClient.TrackEventAsync("replay").Wait();
if (Debugger.IsAttached)
{
TelemetryClient.TrackEventAsync("replay-debug").Wait();
}
Stopwatch watch = new Stopwatch();
watch.Start();
// Set some replay specific options.
Configuration.SchedulingStrategy = "replay";
Configuration.EnableColoredConsoleOutput = true;
Configuration.DisableEnvironmentExit = false;
// Load the configuration of the assembly to be replayed.
LoadAssemblyConfiguration(Configuration.AssemblyToBeAnalyzed);
Console.WriteLine($". Replaying {Configuration.ScheduleFile}");
TestingEngine engine = TestingEngine.Create(Configuration);
engine.Run();
Console.WriteLine(engine.GetReport());
watch.Stop();
if (!Debugger.IsAttached)
{
TelemetryClient.TrackMetricAsync("replay-time", watch.Elapsed.TotalSeconds).Wait();
}
}
/// <summary>
/// Runs the testing engine.
/// </summary>
public void Run()
{
bool isReplaying = this.Strategy is ReplayStrategy;
try
{
TelemetryClient.TrackEventAsync(isReplaying ? "replay" : "test").Wait();
if (Debugger.IsAttached)
{
TelemetryClient.TrackEventAsync(isReplaying ? "replay-debug" : "test-debug").Wait();
}
Task task = this.CreateTestingTask();
if (this.Configuration.TestingTimeout > 0)
{
this.CancellationTokenSource.CancelAfter(
this.Configuration.TestingTimeout * 1000);
}
this.Profiler.StartMeasuringExecutionTime();
if (!this.CancellationTokenSource.IsCancellationRequested)
{
task.Start();
task.Wait(this.CancellationTokenSource.Token);
}
}
catch (OperationCanceledException)
{
if (this.CancellationTokenSource.IsCancellationRequested)
{
this.Logger.WriteLine(LogSeverity.Warning, $"... Task {this.Configuration.TestingProcessId} timed out.");
}
}
catch (AggregateException aex)
{
aex.Handle((ex) =>
{
IO.Debug.WriteLine(ex.Message);
IO.Debug.WriteLine(ex.StackTrace);
return(true);
});
if (aex.InnerException is FileNotFoundException)
{
if (this.Configuration.DisableEnvironmentExit)
{
throw aex.InnerException;
}
else
{
Error.ReportAndExit($"{aex.InnerException.Message}");
}
}
if (this.Configuration.DisableEnvironmentExit)
{
throw aex.InnerException;
}
else
{
Error.ReportAndExit("Exception thrown during testing outside the context of an actor, " +
"possibly in a test method. Please use /debug /v:2 to print more information.");
}
}
catch (Exception ex)
{
this.Logger.WriteLine(LogSeverity.Error, $"... Task {this.Configuration.TestingProcessId} failed due to an internal error: {ex}");
this.TestReport.InternalErrors.Add(ex.ToString());
}
finally
{
this.Profiler.StopMeasuringExecutionTime();
}
if (this.TestReport != null && this.TestReport.NumOfFoundBugs > 0)
{
TelemetryClient.TrackMetricAsync(isReplaying ? "replay-bugs" : "test-bugs", this.TestReport.NumOfFoundBugs).Wait();
}
if (!Debugger.IsAttached)
{
TelemetryClient.TrackMetricAsync(isReplaying ? "replay-time" : "test-time", this.Profiler.Results()).Wait();
}
}
private static void RunTest()
{
if (Configuration.RunAsParallelBugFindingTask)
{
// This is being run as the child test process.
if (Configuration.ParallelDebug)
{
Console.WriteLine("Attach Debugger and press ENTER to continue...");
Console.ReadLine();
}
// Load the configuration of the assembly to be tested.
LoadAssemblyConfiguration(Configuration.AssemblyToBeAnalyzed);
TestingProcess testingProcess = TestingProcess.Create(Configuration);
testingProcess.Run();
return;
}
TelemetryClient = new CoyoteTelemetryClient(Configuration);
TelemetryClient.TrackEventAsync("test").Wait();
if (Debugger.IsAttached)
{
TelemetryClient.TrackEventAsync("test-debug").Wait();
}
Stopwatch watch = new Stopwatch();
watch.Start();
if (Configuration.ReportCodeCoverage || Configuration.ReportActivityCoverage)
{
// This has to be here because both forms of coverage require it.
CodeCoverageInstrumentation.SetOutputDirectory(Configuration, makeHistory: true);
}
if (Configuration.ReportCodeCoverage)
{
// Instruments the program under test for code coverage.
CodeCoverageInstrumentation.Instrument(Configuration);
// Starts monitoring for code coverage.
CodeCoverageMonitor.Start(Configuration);
}
Console.WriteLine(". Testing " + Configuration.AssemblyToBeAnalyzed);
if (!string.IsNullOrEmpty(Configuration.TestMethodName))
{
Console.WriteLine("... Method {0}", Configuration.TestMethodName);
}
// Creates and runs the testing process scheduler.
int bugs = TestingProcessScheduler.Create(Configuration).Run();
if (bugs > 0)
{
TelemetryClient.TrackMetricAsync("test-bugs", bugs).Wait();
}
Console.WriteLine(". Done");
watch.Stop();
if (!Debugger.IsAttached)
{
TelemetryClient.TrackMetricAsync("test-time", watch.Elapsed.TotalSeconds).Wait();
}
}