private static void ReplayTest() { TelemetryClient = new CoyoteTelemetryClient(Configuration); TelemetryClient.TrackEventAsync("replay").Wait(); if (Debugger.IsAttached) { TelemetryClient.TrackEventAsync("replay-debug").Wait(); } Stopwatch watch = new Stopwatch(); watch.Start(); // Set some replay specific options. Configuration.SchedulingStrategy = "replay"; Configuration.EnableColoredConsoleOutput = true; Configuration.DisableEnvironmentExit = false; // Load the configuration of the assembly to be replayed. LoadAssemblyConfiguration(Configuration.AssemblyToBeAnalyzed); Console.WriteLine($". Replaying {Configuration.ScheduleFile}"); TestingEngine engine = TestingEngine.Create(Configuration); engine.Run(); Console.WriteLine(engine.GetReport()); watch.Stop(); if (!Debugger.IsAttached) { TelemetryClient.TrackMetricAsync("replay-time", watch.Elapsed.TotalSeconds).Wait(); } }
/// <summary> /// Initializes a new instance of the <see cref="TestingEngine"/> class. /// </summary> private TestingEngine(Configuration configuration, TestMethodInfo testMethodInfo) { this.Configuration = configuration; this.TestMethodInfo = testMethodInfo; this.DefaultLogger = new ConsoleLogger() { LogLevel = configuration.LogLevel }; this.Logger = this.DefaultLogger; this.Profiler = new Profiler(); this.PerIterationCallbacks = new HashSet <Action <uint> >(); this.TestReport = new TestReport(configuration); this.ReadableTrace = string.Empty; this.ReproducibleTrace = string.Empty; this.CancellationTokenSource = new CancellationTokenSource(); this.PrintGuard = 1; if (configuration.IsDebugVerbosityEnabled) { IO.Debug.IsEnabled = true; } // Do some sanity checking. string error = string.Empty; if (configuration.IsConcurrencyFuzzingEnabled && (configuration.SchedulingStrategy is "replay" || configuration.ScheduleFile.Length > 0)) { error = "Replaying a bug trace is not supported in concurrency fuzzing."; } if (configuration.SchedulingStrategy is "portfolio") { error = "Portfolio testing strategy is only available in parallel testing."; } if (!string.IsNullOrEmpty(error)) { if (configuration.DisableEnvironmentExit) { throw new Exception(error); } else { Error.ReportAndExit(error); } } this.Scheduler = OperationScheduler.Setup(configuration); if (TelemetryClient is null) { TelemetryClient = new CoyoteTelemetryClient(this.Configuration); } }
/// <summary> /// Runs the Coyote testing scheduler. /// </summary> internal void Run() { Console.WriteLine($"... Started the testing task scheduler (process:{Process.GetCurrentProcess().Id})."); // Start the local server. this.StartServer(); this.Profiler.StartMeasuringExecutionTime(); if (this.IsRunOutOfProcess) { using (var telemetryClient = new CoyoteTelemetryClient(this.Configuration)) { telemetryClient.TrackEventAsync("test").Wait(); Stopwatch watch = new Stopwatch(); watch.Start(); this.CreateParallelTestingProcesses(); if (this.Configuration.WaitForTestingProcesses) { this.WaitForParallelTestingProcesses().Wait(); } else { this.RunParallelTestingProcesses(); } watch.Stop(); if (this.GlobalTestReport.NumOfFoundBugs > 0) { telemetryClient.TrackMetricAsync("test-bugs", this.GlobalTestReport.NumOfFoundBugs).Wait(); } if (!Debugger.IsAttached) { telemetryClient.TrackMetricAsync("test-time", watch.Elapsed.TotalSeconds).Wait(); } } } else { this.CreateAndRunInMemoryTestingProcess(); } this.Profiler.StopMeasuringExecutionTime(); // Stop listening and close the server. this.StopServer(); if (!IsProcessCanceled) { // Merges and emits the test report. this.EmitTestReport(); } }
/// <summary> /// Initializes a new instance of the <see cref="TestingEngine"/> class. /// </summary> private TestingEngine(Configuration configuration, TestMethodInfo testMethodInfo) { this.Configuration = configuration; this.TestMethodInfo = testMethodInfo; this.DefaultLogger = new ConsoleLogger() { LogLevel = configuration.LogLevel }; this.Logger = this.DefaultLogger; this.Profiler = new Profiler(); this.PerIterationCallbacks = new HashSet <Action <uint> >(); this.TestReport = new TestReport(configuration); this.ReadableTrace = string.Empty; this.ReproducibleTrace = string.Empty; this.CancellationTokenSource = new CancellationTokenSource(); this.PrintGuard = 1; if (configuration.IsDebugVerbosityEnabled) { IO.Debug.IsEnabled = true; } if (configuration.SchedulingStrategy is "portfolio") { var msg = "Portfolio testing strategy is only " + "available in parallel testing."; if (configuration.DisableEnvironmentExit) { throw new Exception(msg); } else { Error.ReportAndExit(msg); } } this.SchedulingContext = SchedulingContext.Setup(configuration, this.Logger); if (TelemetryClient is null) { TelemetryClient = new CoyoteTelemetryClient(this.Configuration); } }
private static void Main(string[] args) { // Save these so we can force output to happen even if TestingProcess has re-routed it. StdOut = Console.Out; StdError = Console.Error; AppDomain.CurrentDomain.ProcessExit += OnProcessExit; AppDomain.CurrentDomain.UnhandledException += OnUnhandledException; Console.CancelKeyPress += OnProcessCanceled; // Parses the command line options to get the configuration. Configuration = Configuration.Create(); Configuration.TelemetryServerPath = typeof(Program).Assembly.Location; var result = CoyoteTelemetryClient.GetOrCreateMachineId().Result; bool firstTime = result.Item2; var options = new CommandLineOptions(); if (!options.Parse(args, Configuration)) { options.PrintHelp(Console.Out); if (!firstTime && Configuration.EnableTelemetry) { CoyoteTelemetryClient.PrintTelemetryMessage(Console.Out); } Environment.Exit(1); } Configuration.PlatformVersion = GetPlatformVersion(); if (!Configuration.RunAsParallelBugFindingTask) { if (firstTime) { string version = typeof(Microsoft.Coyote.Runtime.CoyoteRuntime).Assembly.GetName().Version.ToString(); Console.WriteLine("Welcome to Microsoft Coyote {0}", version); Console.WriteLine("----------------------------{0}", new string('-', version.Length)); if (Configuration.EnableTelemetry) { CoyoteTelemetryClient.PrintTelemetryMessage(Console.Out); } TelemetryClient = new CoyoteTelemetryClient(Configuration); TelemetryClient.TrackEventAsync("welcome").Wait(); } Console.WriteLine("Microsoft (R) Coyote version {0} for .NET{1}", typeof(CommandLineOptions).Assembly.GetName().Version, GetDotNetVersion()); Console.WriteLine("Copyright (C) Microsoft Corporation. All rights reserved."); Console.WriteLine(); } SetEnvironment(Configuration); switch (Configuration.ToolCommand.ToLower()) { case "test": RunTest(); break; case "replay": ReplayTest(); break; case "rewrite": RewriteAssemblies(); break; case "telemetry": RunServer(); break; } }
/// <summary> /// Initializes a new instance of the <see cref="TestingEngine"/> class. /// </summary> private TestingEngine(Configuration configuration, TestMethodInfo testMethodInfo) { this.Configuration = configuration; this.TestMethodInfo = testMethodInfo; this.DefaultLogger = new ConsoleLogger() { LogLevel = configuration.LogLevel }; this.Logger = this.DefaultLogger; this.Profiler = new Profiler(); this.PerIterationCallbacks = new HashSet <Action <uint> >(); // Initializes scheduling strategy specific components. this.RandomValueGenerator = new RandomValueGenerator(configuration); this.TestReport = new TestReport(configuration); this.ReadableTrace = string.Empty; this.ReproducibleTrace = string.Empty; this.CancellationTokenSource = new CancellationTokenSource(); this.PrintGuard = 1; if (configuration.IsDebugVerbosityEnabled) { IO.Debug.IsEnabled = true; } if (!configuration.UserExplicitlySetLivenessTemperatureThreshold && configuration.MaxFairSchedulingSteps > 0) { configuration.LivenessTemperatureThreshold = configuration.MaxFairSchedulingSteps / 2; } if (configuration.SchedulingStrategy is "replay") { var scheduleDump = this.GetScheduleForReplay(out bool isFair); ScheduleTrace schedule = new ScheduleTrace(scheduleDump); this.Strategy = new ReplayStrategy(configuration, schedule, isFair); } else if (configuration.SchedulingStrategy is "interactive") { configuration.TestingIterations = 1; configuration.PerformFullExploration = false; configuration.IsVerbose = true; this.Strategy = new InteractiveStrategy(configuration, this.Logger); } else if (configuration.SchedulingStrategy is "random") { this.Strategy = new RandomStrategy(configuration.MaxFairSchedulingSteps, this.RandomValueGenerator); } else if (configuration.SchedulingStrategy is "pct") { this.Strategy = new PCTStrategy(configuration.MaxUnfairSchedulingSteps, configuration.StrategyBound, this.RandomValueGenerator); } else if (configuration.SchedulingStrategy is "fairpct") { var prefixLength = configuration.SafetyPrefixBound is 0 ? configuration.MaxUnfairSchedulingSteps : configuration.SafetyPrefixBound; var prefixStrategy = new PCTStrategy(prefixLength, configuration.StrategyBound, this.RandomValueGenerator); var suffixStrategy = new RandomStrategy(configuration.MaxFairSchedulingSteps, this.RandomValueGenerator); this.Strategy = new ComboStrategy(prefixStrategy, suffixStrategy); } else if (configuration.SchedulingStrategy is "probabilistic") { this.Strategy = new ProbabilisticRandomStrategy(configuration.MaxFairSchedulingSteps, configuration.StrategyBound, this.RandomValueGenerator); } else if (configuration.SchedulingStrategy is "dfs") { this.Strategy = new DFSStrategy(configuration.MaxUnfairSchedulingSteps); } else if (configuration.SchedulingStrategy is "rl") { this.Strategy = new QLearningStrategy(configuration.AbstractionLevel, configuration.MaxUnfairSchedulingSteps, this.RandomValueGenerator); } else if (configuration.SchedulingStrategy is "portfolio") { var msg = "Portfolio testing strategy is only " + "available in parallel testing."; if (configuration.DisableEnvironmentExit) { throw new Exception(msg); } else { Error.ReportAndExit(msg); } } if (configuration.SchedulingStrategy != "replay" && configuration.ScheduleFile.Length > 0) { var scheduleDump = this.GetScheduleForReplay(out bool isFair); ScheduleTrace schedule = new ScheduleTrace(scheduleDump); this.Strategy = new ReplayStrategy(configuration, schedule, isFair, this.Strategy); } if (TelemetryClient is null) { TelemetryClient = new CoyoteTelemetryClient(this.Configuration); } }
private static void RunTest() { if (Configuration.RunAsParallelBugFindingTask) { // This is being run as the child test process. if (Configuration.ParallelDebug) { Console.WriteLine("Attach Debugger and press ENTER to continue..."); Console.ReadLine(); } // Load the configuration of the assembly to be tested. LoadAssemblyConfiguration(Configuration.AssemblyToBeAnalyzed); TestingProcess testingProcess = TestingProcess.Create(Configuration); testingProcess.Run(); return; } TelemetryClient = new CoyoteTelemetryClient(Configuration); TelemetryClient.TrackEventAsync("test").Wait(); if (Debugger.IsAttached) { TelemetryClient.TrackEventAsync("test-debug").Wait(); } Stopwatch watch = new Stopwatch(); watch.Start(); if (Configuration.ReportCodeCoverage || Configuration.ReportActivityCoverage) { // This has to be here because both forms of coverage require it. CodeCoverageInstrumentation.SetOutputDirectory(Configuration, makeHistory: true); } if (Configuration.ReportCodeCoverage) { // Instruments the program under test for code coverage. CodeCoverageInstrumentation.Instrument(Configuration); // Starts monitoring for code coverage. CodeCoverageMonitor.Start(Configuration); } Console.WriteLine(". Testing " + Configuration.AssemblyToBeAnalyzed); if (!string.IsNullOrEmpty(Configuration.TestMethodName)) { Console.WriteLine("... Method {0}", Configuration.TestMethodName); } // Creates and runs the testing process scheduler. int bugs = TestingProcessScheduler.Create(Configuration).Run(); if (bugs > 0) { TelemetryClient.TrackMetricAsync("test-bugs", bugs).Wait(); } Console.WriteLine(". Done"); watch.Stop(); if (!Debugger.IsAttached) { TelemetryClient.TrackMetricAsync("test-time", watch.Elapsed.TotalSeconds).Wait(); } }