/// <summary>
/// Invokes the P# testing engine.
/// </summary>
/// <param name="configuration">Configuration</param>
public void Execute(Configuration configuration)
{
// Creates and runs the P# testing engine to find bugs in the P# program.
ITestingEngine testingEngine = TestingEngineFactory.CreateBugFindingEngine(configuration);
var assembly = Assembly.LoadFrom(configuration.AssemblyToBeAnalyzed);
new RaceInstrumentationEngine(testingEngine, configuration);
this.TryLoadReferencedAssemblies(new[] { assembly });
testingEngine.Run();
IO.Error.PrintLine(testingEngine.Report());
if (testingEngine.TestReport.NumOfFoundBugs > 0 ||
configuration.PrintTrace)
{
testingEngine.TryEmitTraces();
}
if (configuration.ReportCodeCoverage)
{
testingEngine.TryEmitCoverageReport();
}
}
public static ITestingEngine RunAndEmitTraces(this ITestingEngine @this, TestArtifact testArtifact)
{
var engine = @this.Run();
@this.TryEmitTraces(testArtifact.Directory, testArtifact.TraceNameBase);
return(@this);
}
public static void Main(string[] args)
{
// Optional: increases verbosity level to see the P# runtime log.
Configuration configuration = Configuration.Create();
configuration.SchedulingIterations = 10;
ITestingEngine engine = TestingEngineFactory.CreateBugFindingEngine(configuration, DefaultImpl.Execute);
engine.Run();
string bug = engine.TestReport.BugReports.FirstOrDefault();
if (bug != null)
{
Console.WriteLine(bug);
}
/*
* // Creates a new P# runtime instance, and passes an optional configuration.
* var runtime = PSharpRuntime.Create(configuration);
*
* // Executes the P# program.
* DefaultImpl.Execute(runtime);
*
* // The P# runtime executes asynchronously, so we wait
* // to not terminate the process.
* Console.WriteLine("Press Enter to terminate...");
*/
}
public static ITestingEngine RunAndEmitTraces(this ITestingEngine @this, MethodBase testMethod, DateTime dateTime)
{
var configuration = @this.TestReport.Configuration;
var testArtifact = configuration.CreateTestArtifact(testMethod, dateTime);
return(@this.RunAndEmitTraces(testArtifact));
}
public static ITestingEngine RunAndEmitTraces(this ITestingEngine @this)
{
var configuration = @this.TestReport.Configuration;
var testArtifact = configuration.CreateCurrentTestArtifact();
return(@this.RunAndEmitTraces(testArtifact));
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="testingEngine">ITestingEngine</param>
/// <param name="configuration">Configuration</param>
public RaceInstrumentationEngine(ITestingEngine testingEngine, Configuration configuration)
: base(new Container(), new EngineOptions(),
new MonitorManager(testingEngine, configuration),
new ThreadMonitorManager(configuration))
{
if (SingletonEngine != null)
{
throw new InvalidOperationException("RaceInstrumentationEngine created more than once.");
}
SingletonEngine = this;
// required?
AppDomain.CurrentDomain.ProcessExit += new EventHandler(CurrentDomainProcessExit);
this.GetService<ISymbolManager>().AddStackFrameFilter(new StackFrameFilter());
if (!ControllerEnvironment.IsMonitoringEnabled)
{
Console.WriteLine("ExtendedReflection monitor not enabled");
throw new NotImplementedException("ExtendedReflection monitor not enabled");
}
((IMonitorManager)this.GetService<MonitorManager>()).RegisterThreadMonitor(
new ThreadMonitorFactory(this.GetService<ThreadMonitorManager>(),
testingEngine, configuration));
((IMonitorManager)this.GetService<MonitorManager>()).RegisterObjectAccessThreadMonitor();
this.ExecutionMonitor.Initialize();
var tid = this.ExecutionMonitor.CreateThread();
_ThreadContext.Start(this.ExecutionMonitor, tid);
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="testingEngine">ITestingEngine</param>
/// <param name="configuration">Configuration</param>
public RaceInstrumentationEngine(ITestingEngine testingEngine, Configuration configuration)
: base(new Container(), new EngineOptions(),
new MonitorManager(testingEngine, configuration),
new ThreadMonitorManager(configuration))
{
if (SingletonEngine != null)
{
throw new InvalidOperationException("RaceInstrumentationEngine created more than once.");
}
SingletonEngine = this;
// required?
AppDomain.CurrentDomain.ProcessExit += new EventHandler(CurrentDomainProcessExit);
this.GetService <ISymbolManager>().AddStackFrameFilter(new StackFrameFilter());
if (!ControllerEnvironment.IsMonitoringEnabled)
{
Console.WriteLine("ExtendedReflection monitor not enabled");
throw new NotImplementedException("ExtendedReflection monitor not enabled");
}
((IMonitorManager)this.GetService <MonitorManager>()).RegisterThreadMonitor(
new ThreadMonitorFactory(this.GetService <ThreadMonitorManager>(),
testingEngine, configuration));
((IMonitorManager)this.GetService <MonitorManager>()).RegisterObjectAccessThreadMonitor();
this.ExecutionMonitor.Initialize();
var tid = this.ExecutionMonitor.CreateThread();
_ThreadContext.Start(this.ExecutionMonitor, tid);
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="host">ICopComponent</param>
/// <param name="configuration">Configuration</param>
public ThreadMonitorFactory(ICopComponent host, ITestingEngine testingEngine,
Configuration configuration)
: base(host)
{
this.Configuration = configuration;
this.TestingEngine = testingEngine;
this.ThreadMonitors = new ThreadMonitorCollection();
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="host">ICopComponent</param>
/// <param name="configuration">Configuration</param>
public ThreadMonitorFactory(ICopComponent host, ITestingEngine testingEngine,
Configuration configuration)
: base(host)
{
this.Configuration = configuration;
this.TestingEngine = testingEngine;
this.ThreadMonitors = new ThreadMonitorCollection();
}
private static void CheckErrors(ITestingEngine engine, IEnumerable <string> expectedErrors)
{
Assert.Equal(1, engine.TestReport.NumOfFoundBugs);
foreach (var bugReport in engine.TestReport.BugReports)
{
var actual = RemoveNonDeterministicValuesFromReport(bugReport);
Assert.Contains(actual, expectedErrors);
}
}
/// <summary>
/// Starts the P# replaying process.
/// </summary>
public void Start()
{
Output.WriteLine(". Reproducing trace in " + this.Configuration.AssemblyToBeAnalyzed);
// Creates a new P# replay engine to reproduce a bug.
ITestingEngine engine = TestingEngineFactory.CreateReplayEngine(this.Configuration);
engine.Run();
Output.WriteLine(engine.Report());
}
public void TestCoverageOnMultipleTests()
{
var configuration = Configuration.Create().WithVerbosityEnabled();
configuration.ReportActivityCoverage = true;
ITestingEngine testingEngine1 = this.Test(r =>
{
var m = r.CreateMachine(typeof(M4));
r.SendEvent(m, new E());
},
configuration);
// Assert that the coverage is as expected.
var coverage1 = testingEngine1.TestReport.CoverageInfo;
Assert.Contains(typeof(M4).FullName, coverage1.MachinesToStates.Keys);
Assert.Contains(typeof(M4.Init).Name, coverage1.MachinesToStates[typeof(M4).FullName]);
Assert.Contains(typeof(M4.Done).Name, coverage1.MachinesToStates[typeof(M4).FullName]);
Assert.Contains(coverage1.RegisteredEvents, tup => tup.Item3 == typeof(E).FullName);
ITestingEngine testingEngine2 = this.Test(r =>
{
var m = r.CreateMachine(typeof(M4));
r.SendEvent(m, new E());
},
configuration);
// Assert that the coverage is the same as before.
var coverage2 = testingEngine2.TestReport.CoverageInfo;
Assert.Contains(typeof(M4).FullName, coverage2.MachinesToStates.Keys);
Assert.Contains(typeof(M4.Init).Name, coverage2.MachinesToStates[typeof(M4).FullName]);
Assert.Contains(typeof(M4.Done).Name, coverage2.MachinesToStates[typeof(M4).FullName]);
Assert.Contains(coverage2.RegisteredEvents, tup => tup.Item3 == typeof(E).FullName);
string coverageReport1, coverageReport2;
var activityCoverageReporter = new ActivityCoverageReporter(coverage1);
using (var writer = new StringWriter())
{
activityCoverageReporter.WriteCoverageText(writer);
coverageReport1 = writer.ToString();
}
activityCoverageReporter = new ActivityCoverageReporter(coverage2);
using (var writer = new StringWriter())
{
activityCoverageReporter.WriteCoverageText(writer);
coverageReport2 = writer.ToString();
}
Assert.Equal(coverageReport1, coverageReport2);
}
protected static string GetBugReport(ITestingEngine engine)
{
string report = string.Empty;
foreach (var bug in engine.TestReport.BugReports)
{
report += bug + "\n";
}
return(report);
}
private static void CheckErrors(ITestingEngine engine, Type exceptionType)
{
var numErrors = engine.TestReport.NumOfFoundBugs;
Assert.Equal(1, numErrors);
var exception = RemoveNonDeterministicValuesFromReport(engine.TestReport.BugReports.First()).
Split(new[] { '\r', '\n' }).FirstOrDefault();
Assert.Contains("'" + exceptionType.ToString() + "'", exception);
}
private string GetBugReport(ITestingEngine engine)
{
string report = "";
foreach (var bug in engine.TestReport.BugReports)
{
report += bug + "\n";
}
return(report);
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="configuration">Configuration</param>
private TestingProcess(Configuration configuration)
{
if (configuration.ParallelBugFindingTasks > 1 &&
configuration.SchedulingStrategy == SchedulingStrategy.Portfolio)
{
TestingPortfolio.ConfigureStrategyForCurrentProcess(configuration);
}
this.Configuration = configuration;
this.TestingEngine = TestingEngineFactory.CreateBugFindingEngine(
this.Configuration);
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="configuration">Configuration</param>
private TestingProcess(Configuration configuration)
{
if (configuration.ParallelBugFindingTasks > 1 &&
configuration.SchedulingStrategy == SchedulingStrategy.Portfolio)
{
TestingPortfolio.ConfigureStrategyForCurrentProcess(configuration);
}
this.Configuration = configuration;
this.TestingEngine = TestingEngineFactory.CreateBugFindingEngine(
this.Configuration);
}
/// <summary>
/// Invokes the P# testing engine.
/// </summary>
/// <param name="configuration">Configuration</param>
public void Execute(Configuration configuration)
{
// Creates and runs the P# testing engine to find bugs in the P# program.
ITestingEngine testingEngine = TestingEngineFactory.CreateBugFindingEngine(configuration);
var assembly = Assembly.LoadFrom(configuration.AssemblyToBeAnalyzed);
new RaceInstrumentationEngine(testingEngine.Reporter, configuration);
this.TryLoadReferencedAssemblies(new[] { assembly });
testingEngine.Run();
Output.WriteLine(testingEngine.Report());
if (testingEngine.TestReport.NumOfFoundBugs > 0)
{
string file = Path.GetFileNameWithoutExtension(configuration.AssemblyToBeAnalyzed);
file += "_" + configuration.TestingProcessId;
string directoryPath;
string suffix = "";
if (configuration.OutputFilePath != "")
{
directoryPath = configuration.OutputFilePath + Path.DirectorySeparatorChar;
}
else
{
var subpath = Path.GetDirectoryName(configuration.AssemblyToBeAnalyzed);
if (subpath == "")
{
subpath = ".";
}
directoryPath = subpath +
Path.DirectorySeparatorChar + "Output" + Path.DirectorySeparatorChar;
}
if (suffix.Length > 0)
{
directoryPath += suffix + Path.DirectorySeparatorChar;
}
Directory.CreateDirectory(directoryPath);
Output.WriteLine($"... Emitting task {configuration.TestingProcessId} traces:");
testingEngine.TryEmitTraces(directoryPath, file);
}
//if (configuration.ReportCodeCoverage)
//{
// testingEngine.TryEmitCoverageReport();
//}
}
/// <summary>
/// Invokes the P# testing engine.
/// </summary>
/// <param name="configuration">Configuration</param>
public void Execute(Configuration configuration)
{
// Creates and runs the P# testing engine to find bugs in the P# program.
ITestingEngine testingEngine = TestingEngineFactory.CreateBugFindingEngine(configuration);
var assembly = Assembly.LoadFrom(configuration.AssemblyToBeAnalyzed);
new RaceInstrumentationEngine(testingEngine, configuration);
this.TryLoadReferencedAssemblies(new[] { assembly });
testingEngine.Run();
}
public void TestUncoveredEvents()
{
var configuration = Configuration.Create().WithVerbosityEnabled();
configuration.ReportActivityCoverage = true;
ITestingEngine testingEngine = this.Test(r =>
{
var m = r.CreateMachine(typeof(M5));
r.SendEvent(m, new E1());
},
configuration);
string result;
var activityCoverageReporter = new ActivityCoverageReporter(testingEngine.TestReport.CoverageInfo);
using (var writer = new StringWriter())
{
activityCoverageReporter.WriteCoverageText(writer);
result = RemoveNamespaceReferencesFromReport(writer.ToString());
result = RemoveExcessiveEmptySpaceFromReport(result);
}
var expected = @"Total event coverage: 50.0%
Machine: M5
***************
Machine event coverage: 50.0%
State: Init
State event coverage: 50.0%
Events received: E1
Events not covered: E2
Next states: Done
State: Done
State event coverage: 100.0%
Previous states: Init
Machine: Env
***************
Machine event coverage: 100.0%
State: Env
State event coverage: 100.0%
Events sent: E1
";
expected = RemoveExcessiveEmptySpaceFromReport(expected);
Assert.Equal(expected, result);
}
public static void Main(string[] args)
{
// Optional: increases verbosity level to see the Coyote runtime log.
Configuration configuration = Configuration.Create();
configuration.SchedulingIterations = 10;
ITestingEngine engine = TestingEngineFactory.CreateBugFindingEngine(configuration, DefaultImpl.Execute);
engine.Run();
string bug = engine.TestReport.BugReports.FirstOrDefault();
if (bug != null)
{
Console.WriteLine(bug);
}
}
private void CheckErrors(ITestingEngine engine, int numExpectedErrors, Func <HashSet <string>, bool> expectedOutputFunc)
{
var numErrors = engine.TestReport.NumOfFoundBugs;
Assert.Equal(numExpectedErrors, numErrors);
var bugReports = new HashSet <string>();
foreach (var bugReport in engine.TestReport.BugReports)
{
var actual = this.RemoveNonDeterministicValuesFromReport(bugReport);
bugReports.Add(actual);
}
Assert.True(expectedOutputFunc(bugReports));
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="configuration">Configuration</param>
private TestingProcess(Configuration configuration)
{
if (configuration.SchedulingStrategy == SchedulingStrategy.Portfolio)
{
TestingPortfolio.ConfigureStrategyForCurrentProcess(configuration);
}
if (configuration.RandomSchedulingSeed != null)
{
configuration.RandomSchedulingSeed = (int)(configuration.RandomSchedulingSeed + (673 * configuration.TestingProcessId));
}
configuration.EnableColoredConsoleOutput = true;
this.Configuration = configuration;
this.TestingEngine = TestingEngineFactory.CreateBugFindingEngine(
this.Configuration);
}
private void CheckErrors(ITestingEngine engine, int numExpectedErrors, ISet <string> expectedOutputs)
{
var numErrors = engine.TestReport.NumOfFoundBugs;
Assert.Equal(numExpectedErrors, numErrors);
if (expectedOutputs.Count > 0)
{
var bugReports = new HashSet <string>();
foreach (var bugReport in engine.TestReport.BugReports)
{
var actual = this.RemoveNonDeterministicValuesFromReport(bugReport);
bugReports.Add(actual);
}
foreach (var expected in expectedOutputs)
{
Assert.Contains(expected, bugReports);
}
}
}
public void TestMachineStateTransitionActivityCoverage()
{
var configuration = Configuration.Create().WithVerbosityEnabled();
configuration.ReportActivityCoverage = true;
ITestingEngine testingEngine = this.Test(r =>
{
r.CreateMachine(typeof(M1));
},
configuration);
string result;
var activityCoverageReporter = new ActivityCoverageReporter(testingEngine.TestReport.CoverageInfo);
using (var writer = new StringWriter())
{
activityCoverageReporter.WriteCoverageText(writer);
result = RemoveNamespaceReferencesFromReport(writer.ToString());
result = RemoveExcessiveEmptySpaceFromReport(result);
}
var expected = @"Total event coverage: 100.0%
Machine: M1
***************
Machine event coverage: 100.0%
State: Init
State event coverage: 100.0%
Next states: Done
State: Done
State event coverage: 100.0%
Previous states: Init
";
expected = RemoveExcessiveEmptySpaceFromReport(expected);
Assert.Equal(expected, result);
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="log">IEventLog</param>
/// <param name="threadIndex">Thread index</param>
/// <param name="callMonitor">IThreadMonitor</param>
/// <param name="testingEngine">ITestingEngine</param>
/// <param name="configuration">Configuration</param>
public ThreadExecutionMonitorDispatcher(IEventLog log, int threadIndex,
IThreadMonitor callMonitor, ITestingEngine testingEngine, Configuration configuration)
: base(threadIndex)
{
SafeDebug.AssertNotNull(callMonitor, "callMonitor");
this.ThreadIndex = threadIndex;
this.Configuration = configuration;
this.ThreadTrace = new List <ThreadTrace>();
this.DebugTrace = new SafeList <string>();
this.CallStack = new SafeStack <Method>();
this.IsDoHandlerCalled = false;
this.IsEntryActionCalled = false;
this.IsExitActionCalled = false;
this.IsAction = false;
this.RecordRW = false;
this.IsCreateMachineMethod = false;
// Registers a callback to emit the thread trace. The callback
// is invoked at the end of each testing iteration.
testingEngine.RegisterPerIterationCallBack(EmitThreadTrace);
}
private static void CheckErrors(ITestingEngine engine, Type exceptionType)
{
Assert.Equal(1, engine.TestReport.NumOfFoundBugs);
Assert.Contains("'" + exceptionType.FullName + "'",
engine.TestReport.BugReports.First().Split(new[] { '\r', '\n' }).FirstOrDefault());
}
public MarshalByRefTestingEngine(ITestingEngine engine)
{
m_engine = engine;
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="testingEngine">ITestingEngine</param>
/// <param name="configuration">Configuration</param>
public MonitorManager(ITestingEngine testingEngine, Configuration configuration)
: base()
{
this.TestingEngine = testingEngine;
this.Configuration = configuration;
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="testingEngine">ITestingEngine</param>
/// <param name="configuration">Configuration</param>
public MonitorManager(ITestingEngine testingEngine, Configuration configuration)
: base()
{
this.TestingEngine = testingEngine;
this.Configuration = configuration;
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="log">IEventLog</param>
/// <param name="threadIndex">Thread index</param>
/// <param name="callMonitor">IThreadMonitor</param>
/// <param name="testingEngine">ITestingEngine</param>
/// <param name="configuration">Configuration</param>
public ThreadExecutionMonitorDispatcher(IEventLog log, int threadIndex,
IThreadMonitor callMonitor, ITestingEngine testingEngine, Configuration configuration)
: base(threadIndex)
{
SafeDebug.AssertNotNull(callMonitor, "callMonitor");
this.ThreadIndex = threadIndex;
this.Configuration = configuration;
this.ThreadTrace = new List<ThreadTrace>();
this.DebugTrace = new SafeList<string>();
this.CallStack = new SafeStack<Method>();
this.IsDoHandlerCalled = false;
this.IsEntryActionCalled = false;
this.IsExitActionCalled = false;
this.IsAction = false;
this.RecordRW = false;
this.IsCreateMachineMethod = false;
// Registers a callback to emit the thread trace. The callback
// is invoked at the end of each testing iteration.
testingEngine.RegisterPerIterationCallBack(EmitThreadTrace);
}
