Beispiel #1
0
        /// <summary>
        /// Initializes a new instance of the <see cref="TestingEngine"/> class.
        /// </summary>
        private TestingEngine(Configuration configuration, TestMethodInfo testMethodInfo)
        {
            this.Configuration  = configuration;
            this.TestMethodInfo = testMethodInfo;

            this.DefaultLogger = new ConsoleLogger()
            {
                LogLevel = configuration.LogLevel
            };
            this.Logger   = this.DefaultLogger;
            this.Profiler = new Profiler();

            this.PerIterationCallbacks = new HashSet <Action <uint> >();

            this.TestReport        = new TestReport(configuration);
            this.ReadableTrace     = string.Empty;
            this.ReproducibleTrace = string.Empty;

            this.CancellationTokenSource = new CancellationTokenSource();
            this.PrintGuard = 1;

            if (configuration.IsDebugVerbosityEnabled)
            {
                IO.Debug.IsEnabled = true;
            }

            // Do some sanity checking.
            string error = string.Empty;

            if (configuration.IsConcurrencyFuzzingEnabled &&
                (configuration.SchedulingStrategy is "replay" || configuration.ScheduleFile.Length > 0))
            {
                error = "Replaying a bug trace is not supported in concurrency fuzzing.";
            }

            if (configuration.SchedulingStrategy is "portfolio")
            {
                error = "Portfolio testing strategy is only available in parallel testing.";
            }

            if (!string.IsNullOrEmpty(error))
            {
                if (configuration.DisableEnvironmentExit)
                {
                    throw new Exception(error);
                }
                else
                {
                    Error.ReportAndExit(error);
                }
            }

            this.Scheduler = OperationScheduler.Setup(configuration);

            if (TelemetryClient is null)
            {
                TelemetryClient = new CoyoteTelemetryClient(this.Configuration);
            }
        }
Beispiel #2
0
 /// <summary>
 /// Creates a new systematic testing engine.
 /// </summary>
 public static TestingEngine Create(Configuration configuration)
 {
     try
     {
         TestMethodInfo testMethodInfo = TestMethodInfo.Create(configuration);
         return(new TestingEngine(configuration, testMethodInfo));
     }
     catch (Exception ex)
     {
         Error.Report(ex.Message);
         throw;
     }
 }
Beispiel #3
0
        /// <summary>
        /// Creates a new systematic testing engine.
        /// </summary>
        public static TestingEngine Create(Configuration configuration, Assembly assembly)
        {
            TestMethodInfo testMethodInfo = null;

            try
            {
                testMethodInfo = TestMethodInfo.GetFromAssembly(assembly, configuration.TestMethodName);
            }
            catch
            {
                Error.ReportAndExit($"Failed to get test method '{configuration.TestMethodName}' from assembly '{assembly.FullName}'");
            }

            return(new TestingEngine(configuration, testMethodInfo));
        }
Beispiel #4
0
        /// <summary>
        /// Creates a new systematic testing engine.
        /// </summary>
        public static TestingEngine Create(Configuration configuration)
        {
            TestMethodInfo testMethodInfo = null;

            try
            {
                testMethodInfo = TestMethodInfo.Create(configuration);
            }
            catch (Exception ex)
            {
                Error.ReportAndExit(ex.Message);
            }

            return(new TestingEngine(configuration, testMethodInfo));
        }
Beispiel #5
0
        /// <summary>
        /// Initializes a new instance of the <see cref="TestingEngine"/> class.
        /// </summary>
        private TestingEngine(Configuration configuration, TestMethodInfo testMethodInfo)
        {
            this.Configuration  = configuration;
            this.TestMethodInfo = testMethodInfo;

            this.DefaultLogger = new ConsoleLogger()
            {
                LogLevel = configuration.LogLevel
            };
            this.Logger   = this.DefaultLogger;
            this.Profiler = new Profiler();

            this.PerIterationCallbacks = new HashSet <Action <uint> >();

            this.TestReport        = new TestReport(configuration);
            this.ReadableTrace     = string.Empty;
            this.ReproducibleTrace = string.Empty;

            this.CancellationTokenSource = new CancellationTokenSource();
            this.PrintGuard = 1;

            if (configuration.IsDebugVerbosityEnabled)
            {
                IO.Debug.IsEnabled = true;
            }

            if (configuration.SchedulingStrategy is "portfolio")
            {
                var msg = "Portfolio testing strategy is only " +
                          "available in parallel testing.";
                if (configuration.DisableEnvironmentExit)
                {
                    throw new Exception(msg);
                }
                else
                {
                    Error.ReportAndExit(msg);
                }
            }

            this.SchedulingContext = SchedulingContext.Setup(configuration, this.Logger);

            if (TelemetryClient is null)
            {
                TelemetryClient = new CoyoteTelemetryClient(this.Configuration);
            }
        }
Beispiel #6
0
        /// <summary>
        /// Initializes a new instance of the <see cref="TestingEngine"/> class.
        /// </summary>
        private TestingEngine(Configuration configuration, TestMethodInfo testMethodInfo)
        {
            this.Configuration  = configuration;
            this.TestMethodInfo = testMethodInfo;

            this.DefaultLogger = new ConsoleLogger()
            {
                LogLevel = configuration.LogLevel
            };
            this.Logger   = this.DefaultLogger;
            this.Profiler = new Profiler();

            this.PerIterationCallbacks = new HashSet <Action <uint> >();

            this.TestReport        = new TestReport(configuration);
            this.ReadableTrace     = string.Empty;
            this.ReproducibleTrace = string.Empty;

            this.CancellationTokenSource = new CancellationTokenSource();
            this.PrintGuard = 1;

            if (configuration.IsDebugVerbosityEnabled)
            {
                IO.Debug.IsEnabled = true;
            }

            // Do some sanity checking.
            string error = string.Empty;

            if (configuration.IsSystematicFuzzingEnabled &&
                (configuration.SchedulingStrategy is "replay" || configuration.ScheduleFile.Length > 0))
            {
                error = "Replaying a bug trace is not currently supported in systematic fuzzing.";
            }

            if (!string.IsNullOrEmpty(error))
            {
                Error.Report(error);
                throw new InvalidOperationException(error);
            }

            this.Scheduler = OperationScheduler.Setup(configuration);

            TelemetryClient = TelemetryClient.GetOrCreate(this.Configuration);
        }
Beispiel #7
0
        /// <summary>
        /// Creates a new systematic testing engine.
        /// </summary>
        public static TestingEngine Create(Configuration configuration)
        {
            TestMethodInfo testMethodInfo = null;

            try
            {
                testMethodInfo = TestMethodInfo.Create(configuration);
            }
            catch (Exception ex)
            {
                if (configuration.DisableEnvironmentExit)
                {
                    throw;
                }
                else
                {
                    Error.ReportAndExit(ex.Message);
                }
            }

            return(new TestingEngine(configuration, testMethodInfo));
        }
Beispiel #8
0
        /// <summary>
        /// Initializes a new instance of the <see cref="TestingEngine"/> class.
        /// </summary>
        private TestingEngine(Configuration configuration, TestMethodInfo testMethodInfo)
        {
            this.Configuration  = configuration;
            this.TestMethodInfo = testMethodInfo;

            this.Logger        = new ConsoleLogger();
            this.ErrorReporter = new ErrorReporter(configuration, this.Logger);
            this.Profiler      = new Profiler();

            this.PerIterationCallbacks = new HashSet <Action <int> >();

            // Initializes scheduling strategy specific components.
            this.RandomValueGenerator = new RandomValueGenerator(configuration);

            this.TestReport        = new TestReport(configuration);
            this.ReadableTrace     = string.Empty;
            this.ReproducableTrace = string.Empty;

            this.CancellationTokenSource = new CancellationTokenSource();
            this.PrintGuard = 1;

            if (!configuration.UserExplicitlySetLivenessTemperatureThreshold &&
                configuration.MaxFairSchedulingSteps > 0)
            {
                configuration.LivenessTemperatureThreshold = configuration.MaxFairSchedulingSteps / 2;
            }

            if (configuration.SchedulingStrategy is "replay")
            {
                var           scheduleDump = this.GetScheduleForReplay(out bool isFair);
                ScheduleTrace schedule     = new ScheduleTrace(scheduleDump);
                this.Strategy = new ReplayStrategy(configuration, schedule, isFair);
            }
            else if (configuration.SchedulingStrategy is "interactive")
            {
                configuration.TestingIterations      = 1;
                configuration.PerformFullExploration = false;
                configuration.IsVerbose = true;
                this.Strategy           = new InteractiveStrategy(configuration, this.Logger);
            }
            else if (configuration.SchedulingStrategy is "random")
            {
                this.Strategy = new RandomStrategy(configuration.MaxFairSchedulingSteps, this.RandomValueGenerator);
            }
            else if (configuration.SchedulingStrategy is "pct")
            {
                this.Strategy = new PCTStrategy(configuration.MaxUnfairSchedulingSteps, configuration.StrategyBound,
                                                this.RandomValueGenerator);
            }
            else if (configuration.SchedulingStrategy is "fairpct")
            {
                var prefixLength = configuration.SafetyPrefixBound == 0 ?
                                   configuration.MaxUnfairSchedulingSteps : configuration.SafetyPrefixBound;
                var prefixStrategy = new PCTStrategy(prefixLength, configuration.StrategyBound, this.RandomValueGenerator);
                var suffixStrategy = new RandomStrategy(configuration.MaxFairSchedulingSteps, this.RandomValueGenerator);
                this.Strategy = new ComboStrategy(prefixStrategy, suffixStrategy);
            }
            else if (configuration.SchedulingStrategy is "probabilistic")
            {
                this.Strategy = new ProbabilisticRandomStrategy(configuration.MaxFairSchedulingSteps,
                                                                configuration.StrategyBound, this.RandomValueGenerator);
            }
            else if (configuration.SchedulingStrategy is "dfs")
            {
                this.Strategy = new DFSStrategy(configuration.MaxUnfairSchedulingSteps);
            }
            else if (configuration.SchedulingStrategy is "portfolio")
            {
                Error.ReportAndExit("Portfolio testing strategy is only " +
                                    "available in parallel testing.");
            }

            if (configuration.SchedulingStrategy != "replay" &&
                configuration.ScheduleFile.Length > 0)
            {
                var           scheduleDump = this.GetScheduleForReplay(out bool isFair);
                ScheduleTrace schedule     = new ScheduleTrace(scheduleDump);
                this.Strategy = new ReplayStrategy(configuration, schedule, isFair, this.Strategy);
            }
        }
Beispiel #9
0
        /// <summary>
        /// Initializes a new instance of the <see cref="TestingEngine"/> class.
        /// </summary>
        private TestingEngine(Configuration configuration, TestMethodInfo testMethodInfo)
        {
            this.Configuration  = configuration;
            this.TestMethodInfo = testMethodInfo;

            this.DefaultLogger = new ConsoleLogger()
            {
                LogLevel = configuration.LogLevel
            };
            this.Logger   = this.DefaultLogger;
            this.Profiler = new Profiler();

            this.PerIterationCallbacks = new HashSet <Action <uint> >();

            // Initializes scheduling strategy specific components.
            this.RandomValueGenerator = new RandomValueGenerator(configuration);

            this.TestReport        = new TestReport(configuration);
            this.ReadableTrace     = string.Empty;
            this.ReproducibleTrace = string.Empty;

            this.CancellationTokenSource = new CancellationTokenSource();
            this.PrintGuard = 1;

            if (configuration.IsDebugVerbosityEnabled)
            {
                IO.Debug.IsEnabled = true;
            }

            if (!configuration.UserExplicitlySetLivenessTemperatureThreshold &&
                configuration.MaxFairSchedulingSteps > 0)
            {
                configuration.LivenessTemperatureThreshold = configuration.MaxFairSchedulingSteps / 2;
            }

            if (configuration.SchedulingStrategy is "replay")
            {
                var           scheduleDump = this.GetScheduleForReplay(out bool isFair);
                ScheduleTrace schedule     = new ScheduleTrace(scheduleDump);
                this.Strategy = new ReplayStrategy(configuration, schedule, isFair);
            }
            else if (configuration.SchedulingStrategy is "interactive")
            {
                configuration.TestingIterations      = 1;
                configuration.PerformFullExploration = false;
                configuration.IsVerbose = true;
                this.Strategy           = new InteractiveStrategy(configuration, this.Logger);
            }
            else if (configuration.SchedulingStrategy is "random")
            {
                this.Strategy = new RandomStrategy(configuration.MaxFairSchedulingSteps, this.RandomValueGenerator);
            }
            else if (configuration.SchedulingStrategy is "pct")
            {
                this.Strategy = new PCTStrategy(configuration.MaxUnfairSchedulingSteps, configuration.StrategyBound,
                                                this.RandomValueGenerator);
            }
            else if (configuration.SchedulingStrategy is "fairpct")
            {
                var prefixLength = configuration.SafetyPrefixBound is 0 ?
                                   configuration.MaxUnfairSchedulingSteps : configuration.SafetyPrefixBound;
                var prefixStrategy = new PCTStrategy(prefixLength, configuration.StrategyBound, this.RandomValueGenerator);
                var suffixStrategy = new RandomStrategy(configuration.MaxFairSchedulingSteps, this.RandomValueGenerator);
                this.Strategy = new ComboStrategy(prefixStrategy, suffixStrategy);
            }
            else if (configuration.SchedulingStrategy is "probabilistic")
            {
                this.Strategy = new ProbabilisticRandomStrategy(configuration.MaxFairSchedulingSteps,
                                                                configuration.StrategyBound, this.RandomValueGenerator);
            }
            else if (configuration.SchedulingStrategy is "dfs")
            {
                this.Strategy = new DFSStrategy(configuration.MaxUnfairSchedulingSteps);
            }
            else if (configuration.SchedulingStrategy is "rl")
            {
                this.Strategy = new QLearningStrategy(configuration.AbstractionLevel, configuration.MaxUnfairSchedulingSteps, this.RandomValueGenerator);
            }
            else if (configuration.SchedulingStrategy is "portfolio")
            {
                var msg = "Portfolio testing strategy is only " +
                          "available in parallel testing.";
                if (configuration.DisableEnvironmentExit)
                {
                    throw new Exception(msg);
                }
                else
                {
                    Error.ReportAndExit(msg);
                }
            }

            if (configuration.SchedulingStrategy != "replay" &&
                configuration.ScheduleFile.Length > 0)
            {
                var           scheduleDump = this.GetScheduleForReplay(out bool isFair);
                ScheduleTrace schedule     = new ScheduleTrace(scheduleDump);
                this.Strategy = new ReplayStrategy(configuration, schedule, isFair, this.Strategy);
            }

            if (TelemetryClient is null)
            {
                TelemetryClient = new CoyoteTelemetryClient(this.Configuration);
            }
        }
Beispiel #10
0
        /// <summary>
        /// Runs the next testing iteration for the specified test method.
        /// </summary>
        private bool RunNextIteration(TestMethodInfo methodInfo, uint iteration)
        {
            if (!this.Scheduler.InitializeNextIteration(iteration))
            {
                // The next iteration cannot run, so stop exploring.
                return(false);
            }

            if (!this.Scheduler.IsReplayingSchedule && this.ShouldPrintIteration(iteration + 1))
            {
                this.Logger.WriteLine(LogSeverity.Important, $"..... Iteration #{iteration + 1}");

                // Flush when logging to console.
                if (this.Logger is ConsoleLogger)
                {
                    Console.Out.Flush();
                }
            }

            // Runtime used to serialize and test the program in this iteration.
            CoyoteRuntime runtime = null;

            // Logger used to intercept the program output if no custom logger
            // is installed and if verbosity is turned off.
            InMemoryLogger runtimeLogger = null;

            // Gets a handle to the standard output and error streams.
            var stdOut = Console.Out;
            var stdErr = Console.Error;

            try
            {
                // Creates a new instance of the controlled runtime.
                runtime = new CoyoteRuntime(this.Configuration, this.Scheduler);

                // If verbosity is turned off, then intercept the program log, and also redirect
                // the standard output and error streams to the runtime logger.
                if (!this.Configuration.IsVerbose)
                {
                    runtimeLogger = new InMemoryLogger();
                    if (this.Logger != this.DefaultLogger)
                    {
                        runtimeLogger.UserLogger = this.Logger;
                    }

                    runtime.Logger = runtimeLogger;

                    Console.SetOut(runtimeLogger.TextWriter);
                    Console.SetError(runtimeLogger.TextWriter);
                }
                else if (this.Logger != this.DefaultLogger)
                {
                    runtime.Logger = this.Logger;
                }

                this.InitializeCustomActorLogging(runtime.DefaultActorExecutionContext);

                // Runs the test and waits for it to terminate.
                Task task = runtime.RunTestAsync(methodInfo.Method, methodInfo.Name);
                task.Wait();

                // Invokes the user-specified iteration disposal method.
                methodInfo.DisposeCurrentIteration();

                // Invoke the per iteration callbacks, if any.
                foreach (var callback in this.PerIterationCallbacks)
                {
                    callback(iteration);
                }

                runtime.LogWriter.LogCompletion();

                this.GatherTestingStatistics(runtime);

                if (!this.Scheduler.IsReplayingSchedule && this.TestReport.NumOfFoundBugs > 0)
                {
                    if (runtimeLogger != null)
                    {
                        this.ReadableTrace = string.Empty;
                        if (this.Configuration.IsTelemetryEnabled)
                        {
                            this.ReadableTrace += $"<TelemetryLog> Anonymized telemetry is enabled, see {LearnAboutTelemetryUrl}.\n";
                        }

                        this.ReadableTrace += runtimeLogger.ToString();
                        this.ReadableTrace += this.TestReport.GetText(this.Configuration, "<StrategyLog>");
                    }

                    if (runtime.SchedulingPolicy is SchedulingPolicy.Interleaving)
                    {
                        this.ReproducibleTrace = this.Scheduler.Trace.Serialize(
                            this.Configuration, this.Scheduler.IsScheduleFair);
                    }
                }
            }
            finally
            {
                if (!this.Configuration.IsVerbose)
                {
                    // Restores the standard output and error streams.
                    Console.SetOut(stdOut);
                    Console.SetError(stdErr);
                }

                if (this.Configuration.IsSystematicFuzzingFallbackEnabled &&
                    runtime.SchedulingPolicy is SchedulingPolicy.Interleaving &&
                    (runtime.ExecutionStatus is ExecutionStatus.ConcurrencyUncontrolled ||
                     runtime.ExecutionStatus is ExecutionStatus.Deadlocked))
                {
                    // Detected uncontrolled concurrency or deadlock, so switch to systematic fuzzing.
                    this.Scheduler = OperationScheduler.Setup(this.Configuration, SchedulingPolicy.Fuzzing,
                                                              this.Scheduler.ValueGenerator);
                    this.Logger.WriteLine(LogSeverity.Important, $"..... Iteration #{iteration + 1} " +
                                          $"enables systematic fuzzing due to uncontrolled concurrency");
                }
                else if (runtime.ExecutionStatus is ExecutionStatus.BoundReached)
                {
                    this.Logger.WriteLine(LogSeverity.Important, $"..... Iteration #{iteration + 1} " +
                                          $"hit bound of '{this.Scheduler.StepCount}' scheduling steps");
                }
                else if (runtime.ExecutionStatus is ExecutionStatus.BugFound)
                {
                    if (!this.Scheduler.IsReplayingSchedule)
                    {
                        this.Logger.WriteLine(LogSeverity.Important, $"..... Iteration #{iteration + 1} " +
                                              $"found bug #{this.TestReport.NumOfFoundBugs}");
                    }

                    this.Logger.WriteLine(LogSeverity.Error, runtime.BugReport);
                }
                else if (this.Scheduler.IsReplayingSchedule)
                {
                    this.Logger.WriteLine(LogSeverity.Error, "Failed to reproduce the bug.");
                }

                // Cleans up the runtime before the next iteration starts.
                runtimeLogger?.Close();
                runtimeLogger?.Dispose();
                runtime?.Dispose();
            }

            return(true);
        }
Beispiel #11
0
        /// <summary>
        /// Creates a new testing task for the specified test method.
        /// </summary>
        private Task CreateTestingTask(TestMethodInfo methodInfo)
        {
            return(new Task(() =>
            {
                this.Logger.WriteLine(LogSeverity.Important, "... Setting up the{0} test:",
                                      string.IsNullOrEmpty(methodInfo.Name) ? string.Empty : $" '{methodInfo.Name}'");
                this.Logger.WriteLine(LogSeverity.Important,
                                      $"..... Using the {this.Scheduler.GetDescription()} exploration strategy.");
                if (this.Configuration.AttachDebugger)
                {
                    this.Logger.WriteLine(LogSeverity.Important,
                                          $"..... Launching and attaching the debugger.");
                    Debugger.Launch();
                }

                try
                {
                    // Invokes the user-specified initialization method.
                    methodInfo.InitializeAllIterations();

                    if (this.Scheduler.IsReplayingSchedule)
                    {
                        this.Logger.WriteLine(LogSeverity.Important, "... Replaying the trace{0}.",
                                              this.Configuration.ScheduleFile.Length > 0 ? $" from {this.Configuration.ScheduleFile}" : string.Empty);
                    }
                    else
                    {
                        this.Logger.WriteLine(LogSeverity.Important, "... Running test iterations:");
                    }

                    uint iteration = 0;
                    while (iteration < this.Configuration.TestingIterations || this.Configuration.TestingTimeout > 0)
                    {
                        if (this.CancellationTokenSource.IsCancellationRequested)
                        {
                            break;
                        }

                        // Runs the next iteration.
                        bool runNext = this.RunNextIteration(methodInfo, iteration);
                        if ((!this.Configuration.RunTestIterationsToCompletion && this.TestReport.NumOfFoundBugs > 0) ||
                            this.Scheduler.IsReplayingSchedule || !runNext)
                        {
                            break;
                        }

                        if (this.Scheduler.ValueGenerator != null && this.Configuration.IsSchedulingSeedIncremental)
                        {
                            // Increments the seed in the random number generator (if one is used), to
                            // capture the seed used by the scheduling strategy in the next iteration.
                            this.Scheduler.ValueGenerator.Seed += 1;
                        }

                        iteration++;
                    }

                    // Invokes the user-specified test disposal method.
                    methodInfo.DisposeAllIterations();
                }
                catch (Exception ex)
                {
                    Exception innerException = ex;
                    while (innerException is TargetInvocationException)
                    {
                        innerException = innerException.InnerException;
                    }

                    if (innerException is AggregateException)
                    {
                        innerException = innerException.InnerException;
                    }

                    if (!(innerException is TaskCanceledException))
                    {
                        ExceptionDispatchInfo.Capture(innerException).Throw();
                    }
                }
            }, this.CancellationTokenSource.Token));
        }