/// <summary> /// Notifies the testing process scheduler /// that a bug was found. /// </summary> /// <param name="processId">Unique process id</param> /// <returns>Boolean value</returns> void ITestingProcessScheduler.NotifyBugFound(uint processId) { lock (this.SchedulerLock) { if (!this.Configuration.PerformFullExploration && this.BugFoundByProcess == null) { Output.WriteLine($"... Task {processId} found a bug."); this.BugFoundByProcess = processId; foreach (var testingProcess in this.TestingProcesses) { if (testingProcess.Key != processId) { this.StopTestingProcess(testingProcess.Key); TestReport testReport = this.GetTestReport(testingProcess.Key); if (testReport != null) { this.MergeTestReport(testReport, testingProcess.Key); } try { this.TestingProcesses[testingProcess.Key].Kill(); this.TestingProcesses[testingProcess.Key].Dispose(); } catch (InvalidOperationException) { IO.Debug.WriteLine("... Unable to terminate testing task " + $"'{testingProcess.Key}'. Task has already terminated."); } } } } } }
/// <summary> /// Emits all the testing coverage related output files. /// </summary> /// <param name="report">TestReport containing CoverageInfo</param> /// <param name="directory">Output directory name, unique for this run</param> /// <param name="file">Output file name</param> private static void EmitTestingCoverageOutputFiles(TestReport report, string directory, string file) { var codeCoverageReporter = new ActivityCoverageReporter(report.CoverageInfo); var filePath = $"{directory}{file}"; string graphFilePath = $"{filePath}.dgml"; Output.WriteLine($"..... Writing {graphFilePath}"); codeCoverageReporter.EmitVisualizationGraph(graphFilePath); string coverageFilePath = $"{filePath}.coverage.txt"; Output.WriteLine($"..... Writing {coverageFilePath}"); codeCoverageReporter.EmitCoverageReport(coverageFilePath); string serFilePath = $"{filePath}.sci"; Output.WriteLine($"..... Writing {serFilePath}"); using (var fs = new FileStream(serFilePath, FileMode.Create)) { var serializer = new DataContractSerializer(typeof(CoverageInfo)); serializer.WriteObject(fs, report.CoverageInfo); } }
/// <summary> /// Constructor. /// </summary> /// <param name="configuration">Configuration</param> private TestingProcessScheduler(Configuration configuration) { this.TestingProcesses = new Dictionary <uint, Process>(); this.TestingProcessChannels = new Dictionary <uint, ITestingProcess>(); this.TestReports = new ConcurrentDictionary <uint, TestReport>(); this.GlobalTestReport = new TestReport(configuration); this.Profiler = new Profiler(); this.SchedulerLock = new object(); this.BugFoundByProcess = null; // Code coverage should be run out-of-process; otherwise VSPerfMon won't shutdown correctly // because an instrumented process (this one) is still running. this.runOutOfProcess = configuration.ParallelBugFindingTasks > 1 || configuration.ReportCodeCoverage; if (configuration.ParallelBugFindingTasks > 1) { configuration.Verbose = 1; configuration.EnableDataRaceDetection = false; } configuration.EnableColoredConsoleOutput = true; this.Configuration = configuration; }
/// <summary> /// Initialized the testing engine. /// </summary> private void Initialize() { this.CancellationTokenSource = new CancellationTokenSource(); this.TestReport = new TestReport(); this.PrintGuard = 1; if (this.Configuration.SchedulingStrategy == SchedulingStrategy.Interactive) { this.Strategy = new InteractiveStrategy(this.Configuration); this.Configuration.SchedulingIterations = 1; this.Configuration.PerformFullExploration = false; this.Configuration.Verbose = 2; } else if (this.Configuration.SchedulingStrategy == SchedulingStrategy.Replay) { string[] scheduleDump = File.ReadAllLines(this.Configuration.ScheduleFile); bool isFair = false; foreach (var line in scheduleDump) { if (!line.StartsWith("--")) { break; } if (line.Equals("--fair-scheduling")) { isFair = true; } else if (line.Equals("--state-caching")) { this.Configuration.CacheProgramState = true; } else if (line.StartsWith("--liveness-temperature-threshold:")) { this.Configuration.LivenessTemperatureThreshold = Int32.Parse(line.Substring("--liveness-temperature-threshold:".Length)); } else if (line.StartsWith("--test-method:")) { this.Configuration.TestMethodName = line.Substring("--test-method:".Length); } } ScheduleTrace schedule = new ScheduleTrace(scheduleDump); this.Strategy = new ReplayStrategy(this.Configuration, schedule, isFair); } else if (this.Configuration.SchedulingStrategy == SchedulingStrategy.Random) { this.Strategy = new RandomStrategy(this.Configuration); } else if (this.Configuration.SchedulingStrategy == SchedulingStrategy.ProbabilisticRandom) { this.Strategy = new ProbabilisticRandomStrategy(this.Configuration, this.Configuration.CoinFlipBound); } else if (this.Configuration.SchedulingStrategy == SchedulingStrategy.DFS) { this.Strategy = new DFSStrategy(this.Configuration); this.Configuration.PerformFullExploration = false; } else if (this.Configuration.SchedulingStrategy == SchedulingStrategy.IDDFS) { this.Strategy = new IterativeDeepeningDFSStrategy(this.Configuration); this.Configuration.PerformFullExploration = false; } else if (this.Configuration.SchedulingStrategy == SchedulingStrategy.DelayBounding) { this.Strategy = new ExhaustiveDelayBoundingStrategy(this.Configuration, this.Configuration.DelayBound); } else if (this.Configuration.SchedulingStrategy == SchedulingStrategy.RandomDelayBounding) { this.Strategy = new RandomDelayBoundingStrategy(this.Configuration, this.Configuration.DelayBound); } else if (this.Configuration.SchedulingStrategy == SchedulingStrategy.PCT) { this.Strategy = new PCTStrategy(this.Configuration, this.Configuration.PrioritySwitchBound); } else if (this.Configuration.SchedulingStrategy == SchedulingStrategy.RandomOperationBounding) { this.Strategy = new RandomOperationBoundingStrategy(this.Configuration); } else if (this.Configuration.SchedulingStrategy == SchedulingStrategy.PrioritizedOperationBounding) { this.Strategy = new PrioritizedOperationBoundingStrategy(this.Configuration, this.Configuration.PrioritySwitchBound); } else if (this.Configuration.SchedulingStrategy == SchedulingStrategy.MaceMC) { this.Strategy = new MaceMCStrategy(this.Configuration); this.Configuration.PerformFullExploration = false; this.Configuration.CacheProgramState = false; } else if (this.Configuration.SchedulingStrategy == SchedulingStrategy.Portfolio) { IO.Error.ReportAndExit("Portfolio testing strategy in only " + "available in parallel testing."); } if (this.Configuration.PrintTrace) { this.Configuration.SchedulingIterations = 1; this.Configuration.PerformFullExploration = false; } this.HasRedirectedConsoleOutput = false; }
/// <summary> /// Creates a bug-reproducing task. /// </summary> /// <returns>Task</returns> private Task CreateBugReproducingTask() { Task task = new Task(() => { // Runtime used to serialize and test the program. BugFindingRuntime runtime = null; // Logger used to intercept the program output if no custom logger // is installed and if verbosity is turned off. InMemoryLogger runtimeLogger = null; // Gets a handle to the standard output and error streams. var stdOut = Console.Out; var stdErr = Console.Error; try { if (base.TestInitMethod != null) { // Initializes the test state. base.TestInitMethod.Invoke(null, new object[] { }); } // Creates a new instance of the bug-finding runtime. if (base.TestRuntimeFactoryMethod != null) { runtime = (BugFindingRuntime)base.TestRuntimeFactoryMethod.Invoke(null, new object[] { base.Configuration, base.Strategy, base.Reporter }); } else { runtime = new BugFindingRuntime(base.Configuration, base.Strategy, base.Reporter); } // If verbosity is turned off, then intercept the program log, and also redirect // the standard output and error streams into the runtime logger. if (base.Configuration.Verbose < 2) { runtimeLogger = new InMemoryLogger(); runtime.SetLogger(runtimeLogger); var writer = new LogWriter(new DisposingLogger()); Console.SetOut(writer); Console.SetError(writer); } // Runs the test inside the P# test-harness machine. runtime.RunTestHarness(base.TestMethod, base.TestAction); // Wait for the test to terminate. runtime.Wait(); // Invokes user-provided cleanup for this iteration. if (base.TestIterationDisposeMethod != null) { // Disposes the test state. base.TestIterationDisposeMethod.Invoke(null, new object[] { }); } // Invokes user-provided cleanup for all iterations. if (base.TestDisposeMethod != null) { // Disposes the test state. base.TestDisposeMethod.Invoke(null, new object[] { }); } this.InternalError = (base.Strategy as ReplayStrategy).ErrorText; // Checks that no monitor is in a hot state at termination. Only // checked if no safety property violations have been found. if (!runtime.Scheduler.BugFound && this.InternalError.Length == 0) { runtime.AssertNoMonitorInHotStateAtTermination(); } if (runtime.Scheduler.BugFound && this.InternalError.Length == 0) { base.ErrorReporter.WriteErrorLine(runtime.Scheduler.BugReport); } TestReport report = runtime.Scheduler.GetReport(); report.CoverageInfo.Merge(runtime.CoverageInfo); this.TestReport.Merge(report); } catch (TargetInvocationException ex) { if (!(ex.InnerException is TaskCanceledException)) { ExceptionDispatchInfo.Capture(ex.InnerException).Throw(); } } finally { if (base.Configuration.Verbose < 2) { // Restores the standard output and error streams. Console.SetOut(stdOut); Console.SetError(stdErr); } // Cleans up the runtime. runtimeLogger?.Dispose(); runtime?.Dispose(); } }, base.CancellationTokenSource.Token); return(task); }