コード例 #1
0
        private static void ReplayTest()
        {
            TelemetryClient = new CoyoteTelemetryClient(Configuration);
            TelemetryClient.TrackEventAsync("replay").Wait();

            if (Debugger.IsAttached)
            {
                TelemetryClient.TrackEventAsync("replay-debug").Wait();
            }

            Stopwatch watch = new Stopwatch();

            watch.Start();

            // Set some replay specific options.
            Configuration.SchedulingStrategy         = "replay";
            Configuration.EnableColoredConsoleOutput = true;
            Configuration.DisableEnvironmentExit     = false;

            // Load the configuration of the assembly to be replayed.
            LoadAssemblyConfiguration(Configuration.AssemblyToBeAnalyzed);

            Console.WriteLine($". Replaying {Configuration.ScheduleFile}");
            TestingEngine engine = TestingEngine.Create(Configuration);

            engine.Run();
            Console.WriteLine(engine.GetReport());

            watch.Stop();

            if (!Debugger.IsAttached)
            {
                TelemetryClient.TrackMetricAsync("replay-time", watch.Elapsed.TotalSeconds).Wait();
            }
        }
コード例 #2
0
ファイル: TestingEngine.cs プロジェクト: Magicianred/coyote
        /// <summary>
        /// Initializes a new instance of the <see cref="TestingEngine"/> class.
        /// </summary>
        private TestingEngine(Configuration configuration, TestMethodInfo testMethodInfo)
        {
            this.Configuration  = configuration;
            this.TestMethodInfo = testMethodInfo;

            this.DefaultLogger = new ConsoleLogger()
            {
                LogLevel = configuration.LogLevel
            };
            this.Logger   = this.DefaultLogger;
            this.Profiler = new Profiler();

            this.PerIterationCallbacks = new HashSet <Action <uint> >();

            this.TestReport        = new TestReport(configuration);
            this.ReadableTrace     = string.Empty;
            this.ReproducibleTrace = string.Empty;

            this.CancellationTokenSource = new CancellationTokenSource();
            this.PrintGuard = 1;

            if (configuration.IsDebugVerbosityEnabled)
            {
                IO.Debug.IsEnabled = true;
            }

            // Do some sanity checking.
            string error = string.Empty;

            if (configuration.IsConcurrencyFuzzingEnabled &&
                (configuration.SchedulingStrategy is "replay" || configuration.ScheduleFile.Length > 0))
            {
                error = "Replaying a bug trace is not supported in concurrency fuzzing.";
            }

            if (configuration.SchedulingStrategy is "portfolio")
            {
                error = "Portfolio testing strategy is only available in parallel testing.";
            }

            if (!string.IsNullOrEmpty(error))
            {
                if (configuration.DisableEnvironmentExit)
                {
                    throw new Exception(error);
                }
                else
                {
                    Error.ReportAndExit(error);
                }
            }

            this.Scheduler = OperationScheduler.Setup(configuration);

            if (TelemetryClient is null)
            {
                TelemetryClient = new CoyoteTelemetryClient(this.Configuration);
            }
        }
コード例 #3
0
        /// <summary>
        /// Runs the Coyote testing scheduler.
        /// </summary>
        internal void Run()
        {
            Console.WriteLine($"... Started the testing task scheduler (process:{Process.GetCurrentProcess().Id}).");

            // Start the local server.
            this.StartServer();

            this.Profiler.StartMeasuringExecutionTime();

            if (this.IsRunOutOfProcess)
            {
                using (var telemetryClient = new CoyoteTelemetryClient(this.Configuration))
                {
                    telemetryClient.TrackEventAsync("test").Wait();

                    Stopwatch watch = new Stopwatch();
                    watch.Start();

                    this.CreateParallelTestingProcesses();
                    if (this.Configuration.WaitForTestingProcesses)
                    {
                        this.WaitForParallelTestingProcesses().Wait();
                    }
                    else
                    {
                        this.RunParallelTestingProcesses();
                    }

                    watch.Stop();

                    if (this.GlobalTestReport.NumOfFoundBugs > 0)
                    {
                        telemetryClient.TrackMetricAsync("test-bugs", this.GlobalTestReport.NumOfFoundBugs).Wait();
                    }

                    if (!Debugger.IsAttached)
                    {
                        telemetryClient.TrackMetricAsync("test-time", watch.Elapsed.TotalSeconds).Wait();
                    }
                }
            }
            else
            {
                this.CreateAndRunInMemoryTestingProcess();
            }

            this.Profiler.StopMeasuringExecutionTime();

            // Stop listening and close the server.
            this.StopServer();

            if (!IsProcessCanceled)
            {
                // Merges and emits the test report.
                this.EmitTestReport();
            }
        }
コード例 #4
0
ファイル: TestingEngine.cs プロジェクト: arunt1204/coyote
        /// <summary>
        /// Initializes a new instance of the <see cref="TestingEngine"/> class.
        /// </summary>
        private TestingEngine(Configuration configuration, TestMethodInfo testMethodInfo)
        {
            this.Configuration  = configuration;
            this.TestMethodInfo = testMethodInfo;

            this.DefaultLogger = new ConsoleLogger()
            {
                LogLevel = configuration.LogLevel
            };
            this.Logger   = this.DefaultLogger;
            this.Profiler = new Profiler();

            this.PerIterationCallbacks = new HashSet <Action <uint> >();

            this.TestReport        = new TestReport(configuration);
            this.ReadableTrace     = string.Empty;
            this.ReproducibleTrace = string.Empty;

            this.CancellationTokenSource = new CancellationTokenSource();
            this.PrintGuard = 1;

            if (configuration.IsDebugVerbosityEnabled)
            {
                IO.Debug.IsEnabled = true;
            }

            if (configuration.SchedulingStrategy is "portfolio")
            {
                var msg = "Portfolio testing strategy is only " +
                          "available in parallel testing.";
                if (configuration.DisableEnvironmentExit)
                {
                    throw new Exception(msg);
                }
                else
                {
                    Error.ReportAndExit(msg);
                }
            }

            this.SchedulingContext = SchedulingContext.Setup(configuration, this.Logger);

            if (TelemetryClient is null)
            {
                TelemetryClient = new CoyoteTelemetryClient(this.Configuration);
            }
        }
コード例 #5
0
        private static void Main(string[] args)
        {
            // Save these so we can force output to happen even if TestingProcess has re-routed it.
            StdOut   = Console.Out;
            StdError = Console.Error;
            AppDomain.CurrentDomain.ProcessExit        += OnProcessExit;
            AppDomain.CurrentDomain.UnhandledException += OnUnhandledException;
            Console.CancelKeyPress += OnProcessCanceled;

            // Parses the command line options to get the configuration.

            Configuration = Configuration.Create();
            Configuration.TelemetryServerPath = typeof(Program).Assembly.Location;

            var  result    = CoyoteTelemetryClient.GetOrCreateMachineId().Result;
            bool firstTime = result.Item2;

            var options = new CommandLineOptions();

            if (!options.Parse(args, Configuration))
            {
                options.PrintHelp(Console.Out);
                if (!firstTime && Configuration.EnableTelemetry)
                {
                    CoyoteTelemetryClient.PrintTelemetryMessage(Console.Out);
                }

                Environment.Exit(1);
            }

            Configuration.PlatformVersion = GetPlatformVersion();

            if (!Configuration.RunAsParallelBugFindingTask)
            {
                if (firstTime)
                {
                    string version = typeof(Microsoft.Coyote.Runtime.CoyoteRuntime).Assembly.GetName().Version.ToString();
                    Console.WriteLine("Welcome to Microsoft Coyote {0}", version);
                    Console.WriteLine("----------------------------{0}", new string('-', version.Length));
                    if (Configuration.EnableTelemetry)
                    {
                        CoyoteTelemetryClient.PrintTelemetryMessage(Console.Out);
                    }

                    TelemetryClient = new CoyoteTelemetryClient(Configuration);
                    TelemetryClient.TrackEventAsync("welcome").Wait();
                }

                Console.WriteLine("Microsoft (R) Coyote version {0} for .NET{1}",
                                  typeof(CommandLineOptions).Assembly.GetName().Version,
                                  GetDotNetVersion());
                Console.WriteLine("Copyright (C) Microsoft Corporation. All rights reserved.");
                Console.WriteLine();
            }

            SetEnvironment(Configuration);

            switch (Configuration.ToolCommand.ToLower())
            {
            case "test":
                RunTest();
                break;

            case "replay":
                ReplayTest();
                break;

            case "rewrite":
                RewriteAssemblies();
                break;

            case "telemetry":
                RunServer();
                break;
            }
        }
コード例 #6
0
ファイル: TestingEngine.cs プロジェクト: arunt1204/coyote-rl
        /// <summary>
        /// Initializes a new instance of the <see cref="TestingEngine"/> class.
        /// </summary>
        private TestingEngine(Configuration configuration, TestMethodInfo testMethodInfo)
        {
            this.Configuration  = configuration;
            this.TestMethodInfo = testMethodInfo;

            this.DefaultLogger = new ConsoleLogger()
            {
                LogLevel = configuration.LogLevel
            };
            this.Logger   = this.DefaultLogger;
            this.Profiler = new Profiler();

            this.PerIterationCallbacks = new HashSet <Action <uint> >();

            // Initializes scheduling strategy specific components.
            this.RandomValueGenerator = new RandomValueGenerator(configuration);

            this.TestReport        = new TestReport(configuration);
            this.ReadableTrace     = string.Empty;
            this.ReproducibleTrace = string.Empty;

            this.CancellationTokenSource = new CancellationTokenSource();
            this.PrintGuard = 1;

            if (configuration.IsDebugVerbosityEnabled)
            {
                IO.Debug.IsEnabled = true;
            }

            if (!configuration.UserExplicitlySetLivenessTemperatureThreshold &&
                configuration.MaxFairSchedulingSteps > 0)
            {
                configuration.LivenessTemperatureThreshold = configuration.MaxFairSchedulingSteps / 2;
            }

            if (configuration.SchedulingStrategy is "replay")
            {
                var           scheduleDump = this.GetScheduleForReplay(out bool isFair);
                ScheduleTrace schedule     = new ScheduleTrace(scheduleDump);
                this.Strategy = new ReplayStrategy(configuration, schedule, isFair);
            }
            else if (configuration.SchedulingStrategy is "interactive")
            {
                configuration.TestingIterations      = 1;
                configuration.PerformFullExploration = false;
                configuration.IsVerbose = true;
                this.Strategy           = new InteractiveStrategy(configuration, this.Logger);
            }
            else if (configuration.SchedulingStrategy is "random")
            {
                this.Strategy = new RandomStrategy(configuration.MaxFairSchedulingSteps, this.RandomValueGenerator);
            }
            else if (configuration.SchedulingStrategy is "pct")
            {
                this.Strategy = new PCTStrategy(configuration.MaxUnfairSchedulingSteps, configuration.StrategyBound,
                                                this.RandomValueGenerator);
            }
            else if (configuration.SchedulingStrategy is "fairpct")
            {
                var prefixLength = configuration.SafetyPrefixBound is 0 ?
                                   configuration.MaxUnfairSchedulingSteps : configuration.SafetyPrefixBound;
                var prefixStrategy = new PCTStrategy(prefixLength, configuration.StrategyBound, this.RandomValueGenerator);
                var suffixStrategy = new RandomStrategy(configuration.MaxFairSchedulingSteps, this.RandomValueGenerator);
                this.Strategy = new ComboStrategy(prefixStrategy, suffixStrategy);
            }
            else if (configuration.SchedulingStrategy is "probabilistic")
            {
                this.Strategy = new ProbabilisticRandomStrategy(configuration.MaxFairSchedulingSteps,
                                                                configuration.StrategyBound, this.RandomValueGenerator);
            }
            else if (configuration.SchedulingStrategy is "dfs")
            {
                this.Strategy = new DFSStrategy(configuration.MaxUnfairSchedulingSteps);
            }
            else if (configuration.SchedulingStrategy is "rl")
            {
                this.Strategy = new QLearningStrategy(configuration.AbstractionLevel, configuration.MaxUnfairSchedulingSteps, this.RandomValueGenerator);
            }
            else if (configuration.SchedulingStrategy is "portfolio")
            {
                var msg = "Portfolio testing strategy is only " +
                          "available in parallel testing.";
                if (configuration.DisableEnvironmentExit)
                {
                    throw new Exception(msg);
                }
                else
                {
                    Error.ReportAndExit(msg);
                }
            }

            if (configuration.SchedulingStrategy != "replay" &&
                configuration.ScheduleFile.Length > 0)
            {
                var           scheduleDump = this.GetScheduleForReplay(out bool isFair);
                ScheduleTrace schedule     = new ScheduleTrace(scheduleDump);
                this.Strategy = new ReplayStrategy(configuration, schedule, isFair, this.Strategy);
            }

            if (TelemetryClient is null)
            {
                TelemetryClient = new CoyoteTelemetryClient(this.Configuration);
            }
        }
コード例 #7
0
        private static void RunTest()
        {
            if (Configuration.RunAsParallelBugFindingTask)
            {
                // This is being run as the child test process.
                if (Configuration.ParallelDebug)
                {
                    Console.WriteLine("Attach Debugger and press ENTER to continue...");
                    Console.ReadLine();
                }

                // Load the configuration of the assembly to be tested.
                LoadAssemblyConfiguration(Configuration.AssemblyToBeAnalyzed);

                TestingProcess testingProcess = TestingProcess.Create(Configuration);
                testingProcess.Run();
                return;
            }

            TelemetryClient = new CoyoteTelemetryClient(Configuration);
            TelemetryClient.TrackEventAsync("test").Wait();

            if (Debugger.IsAttached)
            {
                TelemetryClient.TrackEventAsync("test-debug").Wait();
            }

            Stopwatch watch = new Stopwatch();

            watch.Start();

            if (Configuration.ReportCodeCoverage || Configuration.ReportActivityCoverage)
            {
                // This has to be here because both forms of coverage require it.
                CodeCoverageInstrumentation.SetOutputDirectory(Configuration, makeHistory: true);
            }

            if (Configuration.ReportCodeCoverage)
            {
                // Instruments the program under test for code coverage.
                CodeCoverageInstrumentation.Instrument(Configuration);

                // Starts monitoring for code coverage.
                CodeCoverageMonitor.Start(Configuration);
            }

            Console.WriteLine(". Testing " + Configuration.AssemblyToBeAnalyzed);
            if (!string.IsNullOrEmpty(Configuration.TestMethodName))
            {
                Console.WriteLine("... Method {0}", Configuration.TestMethodName);
            }

            // Creates and runs the testing process scheduler.
            int bugs = TestingProcessScheduler.Create(Configuration).Run();

            if (bugs > 0)
            {
                TelemetryClient.TrackMetricAsync("test-bugs", bugs).Wait();
            }

            Console.WriteLine(". Done");

            watch.Stop();
            if (!Debugger.IsAttached)
            {
                TelemetryClient.TrackMetricAsync("test-time", watch.Elapsed.TotalSeconds).Wait();
            }
        }