/// <summary> /// Invokes the P# testing engine. /// </summary> /// <param name="configuration">Configuration</param> public void Execute(Configuration configuration) { // Creates and runs the P# testing engine to find bugs in the P# program. ITestingEngine testingEngine = TestingEngineFactory.CreateBugFindingEngine(configuration); var assembly = Assembly.LoadFrom(configuration.AssemblyToBeAnalyzed); new RaceInstrumentationEngine(testingEngine, configuration); this.TryLoadReferencedAssemblies(new[] { assembly }); testingEngine.Run(); IO.Error.PrintLine(testingEngine.Report()); if (testingEngine.TestReport.NumOfFoundBugs > 0 || configuration.PrintTrace) { testingEngine.TryEmitTraces(); } if (configuration.ReportCodeCoverage) { testingEngine.TryEmitCoverageReport(); } }
public static ITestingEngine RunAndEmitTraces(this ITestingEngine @this, TestArtifact testArtifact) { var engine = @this.Run(); @this.TryEmitTraces(testArtifact.Directory, testArtifact.TraceNameBase); return(@this); }
public static void Main(string[] args) { // Optional: increases verbosity level to see the P# runtime log. Configuration configuration = Configuration.Create(); configuration.SchedulingIterations = 10; ITestingEngine engine = TestingEngineFactory.CreateBugFindingEngine(configuration, DefaultImpl.Execute); engine.Run(); string bug = engine.TestReport.BugReports.FirstOrDefault(); if (bug != null) { Console.WriteLine(bug); } /* * // Creates a new P# runtime instance, and passes an optional configuration. * var runtime = PSharpRuntime.Create(configuration); * * // Executes the P# program. * DefaultImpl.Execute(runtime); * * // The P# runtime executes asynchronously, so we wait * // to not terminate the process. * Console.WriteLine("Press Enter to terminate..."); */ }
public static ITestingEngine RunAndEmitTraces(this ITestingEngine @this, MethodBase testMethod, DateTime dateTime) { var configuration = @this.TestReport.Configuration; var testArtifact = configuration.CreateTestArtifact(testMethod, dateTime); return(@this.RunAndEmitTraces(testArtifact)); }
public static ITestingEngine RunAndEmitTraces(this ITestingEngine @this) { var configuration = @this.TestReport.Configuration; var testArtifact = configuration.CreateCurrentTestArtifact(); return(@this.RunAndEmitTraces(testArtifact)); }
/// <summary> /// Constructor. /// </summary> /// <param name="testingEngine">ITestingEngine</param> /// <param name="configuration">Configuration</param> public RaceInstrumentationEngine(ITestingEngine testingEngine, Configuration configuration) : base(new Container(), new EngineOptions(), new MonitorManager(testingEngine, configuration), new ThreadMonitorManager(configuration)) { if (SingletonEngine != null) { throw new InvalidOperationException("RaceInstrumentationEngine created more than once."); } SingletonEngine = this; // required? AppDomain.CurrentDomain.ProcessExit += new EventHandler(CurrentDomainProcessExit); this.GetService<ISymbolManager>().AddStackFrameFilter(new StackFrameFilter()); if (!ControllerEnvironment.IsMonitoringEnabled) { Console.WriteLine("ExtendedReflection monitor not enabled"); throw new NotImplementedException("ExtendedReflection monitor not enabled"); } ((IMonitorManager)this.GetService<MonitorManager>()).RegisterThreadMonitor( new ThreadMonitorFactory(this.GetService<ThreadMonitorManager>(), testingEngine, configuration)); ((IMonitorManager)this.GetService<MonitorManager>()).RegisterObjectAccessThreadMonitor(); this.ExecutionMonitor.Initialize(); var tid = this.ExecutionMonitor.CreateThread(); _ThreadContext.Start(this.ExecutionMonitor, tid); }
/// <summary> /// Constructor. /// </summary> /// <param name="testingEngine">ITestingEngine</param> /// <param name="configuration">Configuration</param> public RaceInstrumentationEngine(ITestingEngine testingEngine, Configuration configuration) : base(new Container(), new EngineOptions(), new MonitorManager(testingEngine, configuration), new ThreadMonitorManager(configuration)) { if (SingletonEngine != null) { throw new InvalidOperationException("RaceInstrumentationEngine created more than once."); } SingletonEngine = this; // required? AppDomain.CurrentDomain.ProcessExit += new EventHandler(CurrentDomainProcessExit); this.GetService <ISymbolManager>().AddStackFrameFilter(new StackFrameFilter()); if (!ControllerEnvironment.IsMonitoringEnabled) { Console.WriteLine("ExtendedReflection monitor not enabled"); throw new NotImplementedException("ExtendedReflection monitor not enabled"); } ((IMonitorManager)this.GetService <MonitorManager>()).RegisterThreadMonitor( new ThreadMonitorFactory(this.GetService <ThreadMonitorManager>(), testingEngine, configuration)); ((IMonitorManager)this.GetService <MonitorManager>()).RegisterObjectAccessThreadMonitor(); this.ExecutionMonitor.Initialize(); var tid = this.ExecutionMonitor.CreateThread(); _ThreadContext.Start(this.ExecutionMonitor, tid); }
/// <summary> /// Constructor. /// </summary> /// <param name="host">ICopComponent</param> /// <param name="configuration">Configuration</param> public ThreadMonitorFactory(ICopComponent host, ITestingEngine testingEngine, Configuration configuration) : base(host) { this.Configuration = configuration; this.TestingEngine = testingEngine; this.ThreadMonitors = new ThreadMonitorCollection(); }
private static void CheckErrors(ITestingEngine engine, IEnumerable <string> expectedErrors) { Assert.Equal(1, engine.TestReport.NumOfFoundBugs); foreach (var bugReport in engine.TestReport.BugReports) { var actual = RemoveNonDeterministicValuesFromReport(bugReport); Assert.Contains(actual, expectedErrors); } }
/// <summary> /// Starts the P# replaying process. /// </summary> public void Start() { Output.WriteLine(". Reproducing trace in " + this.Configuration.AssemblyToBeAnalyzed); // Creates a new P# replay engine to reproduce a bug. ITestingEngine engine = TestingEngineFactory.CreateReplayEngine(this.Configuration); engine.Run(); Output.WriteLine(engine.Report()); }
public void TestCoverageOnMultipleTests() { var configuration = Configuration.Create().WithVerbosityEnabled(); configuration.ReportActivityCoverage = true; ITestingEngine testingEngine1 = this.Test(r => { var m = r.CreateMachine(typeof(M4)); r.SendEvent(m, new E()); }, configuration); // Assert that the coverage is as expected. var coverage1 = testingEngine1.TestReport.CoverageInfo; Assert.Contains(typeof(M4).FullName, coverage1.MachinesToStates.Keys); Assert.Contains(typeof(M4.Init).Name, coverage1.MachinesToStates[typeof(M4).FullName]); Assert.Contains(typeof(M4.Done).Name, coverage1.MachinesToStates[typeof(M4).FullName]); Assert.Contains(coverage1.RegisteredEvents, tup => tup.Item3 == typeof(E).FullName); ITestingEngine testingEngine2 = this.Test(r => { var m = r.CreateMachine(typeof(M4)); r.SendEvent(m, new E()); }, configuration); // Assert that the coverage is the same as before. var coverage2 = testingEngine2.TestReport.CoverageInfo; Assert.Contains(typeof(M4).FullName, coverage2.MachinesToStates.Keys); Assert.Contains(typeof(M4.Init).Name, coverage2.MachinesToStates[typeof(M4).FullName]); Assert.Contains(typeof(M4.Done).Name, coverage2.MachinesToStates[typeof(M4).FullName]); Assert.Contains(coverage2.RegisteredEvents, tup => tup.Item3 == typeof(E).FullName); string coverageReport1, coverageReport2; var activityCoverageReporter = new ActivityCoverageReporter(coverage1); using (var writer = new StringWriter()) { activityCoverageReporter.WriteCoverageText(writer); coverageReport1 = writer.ToString(); } activityCoverageReporter = new ActivityCoverageReporter(coverage2); using (var writer = new StringWriter()) { activityCoverageReporter.WriteCoverageText(writer); coverageReport2 = writer.ToString(); } Assert.Equal(coverageReport1, coverageReport2); }
protected static string GetBugReport(ITestingEngine engine) { string report = string.Empty; foreach (var bug in engine.TestReport.BugReports) { report += bug + "\n"; } return(report); }
private static void CheckErrors(ITestingEngine engine, Type exceptionType) { var numErrors = engine.TestReport.NumOfFoundBugs; Assert.Equal(1, numErrors); var exception = RemoveNonDeterministicValuesFromReport(engine.TestReport.BugReports.First()). Split(new[] { '\r', '\n' }).FirstOrDefault(); Assert.Contains("'" + exceptionType.ToString() + "'", exception); }
private string GetBugReport(ITestingEngine engine) { string report = ""; foreach (var bug in engine.TestReport.BugReports) { report += bug + "\n"; } return(report); }
/// <summary> /// Constructor. /// </summary> /// <param name="configuration">Configuration</param> private TestingProcess(Configuration configuration) { if (configuration.ParallelBugFindingTasks > 1 && configuration.SchedulingStrategy == SchedulingStrategy.Portfolio) { TestingPortfolio.ConfigureStrategyForCurrentProcess(configuration); } this.Configuration = configuration; this.TestingEngine = TestingEngineFactory.CreateBugFindingEngine( this.Configuration); }
/// <summary> /// Invokes the P# testing engine. /// </summary> /// <param name="configuration">Configuration</param> public void Execute(Configuration configuration) { // Creates and runs the P# testing engine to find bugs in the P# program. ITestingEngine testingEngine = TestingEngineFactory.CreateBugFindingEngine(configuration); var assembly = Assembly.LoadFrom(configuration.AssemblyToBeAnalyzed); new RaceInstrumentationEngine(testingEngine.Reporter, configuration); this.TryLoadReferencedAssemblies(new[] { assembly }); testingEngine.Run(); Output.WriteLine(testingEngine.Report()); if (testingEngine.TestReport.NumOfFoundBugs > 0) { string file = Path.GetFileNameWithoutExtension(configuration.AssemblyToBeAnalyzed); file += "_" + configuration.TestingProcessId; string directoryPath; string suffix = ""; if (configuration.OutputFilePath != "") { directoryPath = configuration.OutputFilePath + Path.DirectorySeparatorChar; } else { var subpath = Path.GetDirectoryName(configuration.AssemblyToBeAnalyzed); if (subpath == "") { subpath = "."; } directoryPath = subpath + Path.DirectorySeparatorChar + "Output" + Path.DirectorySeparatorChar; } if (suffix.Length > 0) { directoryPath += suffix + Path.DirectorySeparatorChar; } Directory.CreateDirectory(directoryPath); Output.WriteLine($"... Emitting task {configuration.TestingProcessId} traces:"); testingEngine.TryEmitTraces(directoryPath, file); } //if (configuration.ReportCodeCoverage) //{ // testingEngine.TryEmitCoverageReport(); //} }
/// <summary> /// Invokes the P# testing engine. /// </summary> /// <param name="configuration">Configuration</param> public void Execute(Configuration configuration) { // Creates and runs the P# testing engine to find bugs in the P# program. ITestingEngine testingEngine = TestingEngineFactory.CreateBugFindingEngine(configuration); var assembly = Assembly.LoadFrom(configuration.AssemblyToBeAnalyzed); new RaceInstrumentationEngine(testingEngine, configuration); this.TryLoadReferencedAssemblies(new[] { assembly }); testingEngine.Run(); }
public void TestUncoveredEvents() { var configuration = Configuration.Create().WithVerbosityEnabled(); configuration.ReportActivityCoverage = true; ITestingEngine testingEngine = this.Test(r => { var m = r.CreateMachine(typeof(M5)); r.SendEvent(m, new E1()); }, configuration); string result; var activityCoverageReporter = new ActivityCoverageReporter(testingEngine.TestReport.CoverageInfo); using (var writer = new StringWriter()) { activityCoverageReporter.WriteCoverageText(writer); result = RemoveNamespaceReferencesFromReport(writer.ToString()); result = RemoveExcessiveEmptySpaceFromReport(result); } var expected = @"Total event coverage: 50.0% Machine: M5 *************** Machine event coverage: 50.0% State: Init State event coverage: 50.0% Events received: E1 Events not covered: E2 Next states: Done State: Done State event coverage: 100.0% Previous states: Init Machine: Env *************** Machine event coverage: 100.0% State: Env State event coverage: 100.0% Events sent: E1 "; expected = RemoveExcessiveEmptySpaceFromReport(expected); Assert.Equal(expected, result); }
public static void Main(string[] args) { // Optional: increases verbosity level to see the Coyote runtime log. Configuration configuration = Configuration.Create(); configuration.SchedulingIterations = 10; ITestingEngine engine = TestingEngineFactory.CreateBugFindingEngine(configuration, DefaultImpl.Execute); engine.Run(); string bug = engine.TestReport.BugReports.FirstOrDefault(); if (bug != null) { Console.WriteLine(bug); } }
private void CheckErrors(ITestingEngine engine, int numExpectedErrors, Func <HashSet <string>, bool> expectedOutputFunc) { var numErrors = engine.TestReport.NumOfFoundBugs; Assert.Equal(numExpectedErrors, numErrors); var bugReports = new HashSet <string>(); foreach (var bugReport in engine.TestReport.BugReports) { var actual = this.RemoveNonDeterministicValuesFromReport(bugReport); bugReports.Add(actual); } Assert.True(expectedOutputFunc(bugReports)); }
/// <summary> /// Constructor. /// </summary> /// <param name="configuration">Configuration</param> private TestingProcess(Configuration configuration) { if (configuration.SchedulingStrategy == SchedulingStrategy.Portfolio) { TestingPortfolio.ConfigureStrategyForCurrentProcess(configuration); } if (configuration.RandomSchedulingSeed != null) { configuration.RandomSchedulingSeed = (int)(configuration.RandomSchedulingSeed + (673 * configuration.TestingProcessId)); } configuration.EnableColoredConsoleOutput = true; this.Configuration = configuration; this.TestingEngine = TestingEngineFactory.CreateBugFindingEngine( this.Configuration); }
private void CheckErrors(ITestingEngine engine, int numExpectedErrors, ISet <string> expectedOutputs) { var numErrors = engine.TestReport.NumOfFoundBugs; Assert.Equal(numExpectedErrors, numErrors); if (expectedOutputs.Count > 0) { var bugReports = new HashSet <string>(); foreach (var bugReport in engine.TestReport.BugReports) { var actual = this.RemoveNonDeterministicValuesFromReport(bugReport); bugReports.Add(actual); } foreach (var expected in expectedOutputs) { Assert.Contains(expected, bugReports); } } }
public void TestMachineStateTransitionActivityCoverage() { var configuration = Configuration.Create().WithVerbosityEnabled(); configuration.ReportActivityCoverage = true; ITestingEngine testingEngine = this.Test(r => { r.CreateMachine(typeof(M1)); }, configuration); string result; var activityCoverageReporter = new ActivityCoverageReporter(testingEngine.TestReport.CoverageInfo); using (var writer = new StringWriter()) { activityCoverageReporter.WriteCoverageText(writer); result = RemoveNamespaceReferencesFromReport(writer.ToString()); result = RemoveExcessiveEmptySpaceFromReport(result); } var expected = @"Total event coverage: 100.0% Machine: M1 *************** Machine event coverage: 100.0% State: Init State event coverage: 100.0% Next states: Done State: Done State event coverage: 100.0% Previous states: Init "; expected = RemoveExcessiveEmptySpaceFromReport(expected); Assert.Equal(expected, result); }
/// <summary> /// Constructor. /// </summary> /// <param name="log">IEventLog</param> /// <param name="threadIndex">Thread index</param> /// <param name="callMonitor">IThreadMonitor</param> /// <param name="testingEngine">ITestingEngine</param> /// <param name="configuration">Configuration</param> public ThreadExecutionMonitorDispatcher(IEventLog log, int threadIndex, IThreadMonitor callMonitor, ITestingEngine testingEngine, Configuration configuration) : base(threadIndex) { SafeDebug.AssertNotNull(callMonitor, "callMonitor"); this.ThreadIndex = threadIndex; this.Configuration = configuration; this.ThreadTrace = new List <ThreadTrace>(); this.DebugTrace = new SafeList <string>(); this.CallStack = new SafeStack <Method>(); this.IsDoHandlerCalled = false; this.IsEntryActionCalled = false; this.IsExitActionCalled = false; this.IsAction = false; this.RecordRW = false; this.IsCreateMachineMethod = false; // Registers a callback to emit the thread trace. The callback // is invoked at the end of each testing iteration. testingEngine.RegisterPerIterationCallBack(EmitThreadTrace); }
private static void CheckErrors(ITestingEngine engine, Type exceptionType) { Assert.Equal(1, engine.TestReport.NumOfFoundBugs); Assert.Contains("'" + exceptionType.FullName + "'", engine.TestReport.BugReports.First().Split(new[] { '\r', '\n' }).FirstOrDefault()); }
public MarshalByRefTestingEngine(ITestingEngine engine) { m_engine = engine; }
/// <summary> /// Constructor. /// </summary> /// <param name="testingEngine">ITestingEngine</param> /// <param name="configuration">Configuration</param> public MonitorManager(ITestingEngine testingEngine, Configuration configuration) : base() { this.TestingEngine = testingEngine; this.Configuration = configuration; }
/// <summary> /// Constructor. /// </summary> /// <param name="log">IEventLog</param> /// <param name="threadIndex">Thread index</param> /// <param name="callMonitor">IThreadMonitor</param> /// <param name="testingEngine">ITestingEngine</param> /// <param name="configuration">Configuration</param> public ThreadExecutionMonitorDispatcher(IEventLog log, int threadIndex, IThreadMonitor callMonitor, ITestingEngine testingEngine, Configuration configuration) : base(threadIndex) { SafeDebug.AssertNotNull(callMonitor, "callMonitor"); this.ThreadIndex = threadIndex; this.Configuration = configuration; this.ThreadTrace = new List<ThreadTrace>(); this.DebugTrace = new SafeList<string>(); this.CallStack = new SafeStack<Method>(); this.IsDoHandlerCalled = false; this.IsEntryActionCalled = false; this.IsExitActionCalled = false; this.IsAction = false; this.RecordRW = false; this.IsCreateMachineMethod = false; // Registers a callback to emit the thread trace. The callback // is invoked at the end of each testing iteration. testingEngine.RegisterPerIterationCallBack(EmitThreadTrace); }