Exemplo n.º 1
0
        /// <summary>
        /// Gathers the exploration strategy statistics for the latest testing iteration.
        /// </summary>
        private void GatherIterationStatistics(SystematicTestingRuntime runtime)
        {
            TestReport report = runtime.Scheduler.GetReport();

            report.CoverageInfo.Merge(runtime.CoverageInfo);
            this.TestReport.Merge(report);
        }
Exemplo n.º 2
0
        /// <summary>
        /// Initializes a new instance of the <see cref="MockSharedCounter"/> class.
        /// </summary>
        public MockSharedCounter(int value, SystematicTestingRuntime runtime)
        {
            this.Runtime        = runtime;
            this.CounterMachine = this.Runtime.CreateMachine(typeof(SharedCounterMachine));
            var currentMachine = this.Runtime.GetExecutingMachine <Machine>();

            this.Runtime.SendEvent(this.CounterMachine, SharedCounterEvent.SetEvent(currentMachine.Id, value));
            currentMachine.Receive(typeof(SharedCounterResponseEvent)).Wait();
        }
Exemplo n.º 3
0
 /// <summary>
 /// Initializes a new instance of the <see cref="SerializedMachineStateManager"/> class.
 /// </summary>
 internal SerializedMachineStateManager(SystematicTestingRuntime runtime, Machine machine, Guid operationGroupId)
 {
     this.Runtime = runtime;
     this.Machine = machine;
     this.IsEventHandlerRunning = true;
     this.OperationGroupId      = operationGroupId;
     this.ProgramCounter        = 0;
     this.IsTransitionStatementCalledInCurrentAction = false;
     this.IsInsideOnExit = false;
 }
Exemplo n.º 4
0
 /// <summary>
 /// Initializes a new instance of the <see cref="BugFindingScheduler"/> class.
 /// </summary>
 internal BugFindingScheduler(SystematicTestingRuntime runtime, ISchedulingStrategy strategy)
 {
     this.Runtime                  = runtime;
     this.Strategy                 = strategy;
     this.OperationMap             = new Dictionary <ulong, AsyncOperation>();
     this.CompletionSource         = new TaskCompletionSource <bool>();
     this.IsSchedulerRunning       = true;
     this.BugFound                 = false;
     this.HasFullyExploredSchedule = false;
 }
Exemplo n.º 5
0
        /// <summary>
        /// Constructs a reproducable trace.
        /// </summary>
        private void ConstructReproducableTrace(SystematicTestingRuntime runtime)
        {
            StringBuilder stringBuilder = new StringBuilder();

            if (this.Strategy.IsFair())
            {
                stringBuilder.Append("--fair-scheduling").Append(Environment.NewLine);
            }

            if (this.Configuration.EnableCycleDetection)
            {
                stringBuilder.Append("--cycle-detection").Append(Environment.NewLine);
                stringBuilder.Append("--liveness-temperature-threshold:" +
                                     this.Configuration.LivenessTemperatureThreshold).
                Append(Environment.NewLine);
            }
            else
            {
                stringBuilder.Append("--liveness-temperature-threshold:" +
                                     this.Configuration.LivenessTemperatureThreshold).
                Append(Environment.NewLine);
            }

            if (!string.IsNullOrEmpty(this.Configuration.TestMethodName))
            {
                stringBuilder.Append("--test-method:" +
                                     this.Configuration.TestMethodName).
                Append(Environment.NewLine);
            }

            for (int idx = 0; idx < runtime.ScheduleTrace.Count; idx++)
            {
                ScheduleStep step = runtime.ScheduleTrace[idx];
                if (step.Type == ScheduleStepType.SchedulingChoice)
                {
                    stringBuilder.Append($"({step.ScheduledOperationId})");
                }
                else if (step.BooleanChoice != null)
                {
                    stringBuilder.Append(step.BooleanChoice.Value);
                }
                else
                {
                    stringBuilder.Append(step.IntegerChoice.Value);
                }

                if (idx < runtime.ScheduleTrace.Count - 1)
                {
                    stringBuilder.Append(Environment.NewLine);
                }
            }

            this.ReproducableTrace = stringBuilder.ToString();
        }
Exemplo n.º 6
0
 /// <summary>
 /// Initializes a new instance of the <see cref="MockSharedDictionary{TKey, TValue}"/> class.
 /// </summary>
 public MockSharedDictionary(IEqualityComparer <TKey> comparer, SystematicTestingRuntime runtime)
 {
     this.Runtime = runtime;
     if (comparer != null)
     {
         this.DictionaryMachine = this.Runtime.CreateMachine(
             typeof(SharedDictionaryMachine <TKey, TValue>),
             SharedDictionaryEvent.InitEvent(comparer));
     }
     else
     {
         this.DictionaryMachine = this.Runtime.CreateMachine(typeof(SharedDictionaryMachine <TKey, TValue>));
     }
 }
Exemplo n.º 7
0
        /// <summary>
        /// Runs the next testing iteration.
        /// </summary>
        private void RunNextIteration(int iteration)
        {
            if (this.ShouldPrintIteration(iteration + 1))
            {
                this.Logger.WriteLine($"..... Iteration #{iteration + 1}");

                // Flush when logging to console.
                if (this.Logger is ConsoleLogger)
                {
                    Console.Out.Flush();
                }
            }

            // Runtime used to serialize and test the program in this iteration.
            SystematicTestingRuntime runtime = null;

            // Logger used to intercept the program output if no custom logger
            // is installed and if verbosity is turned off.
            InMemoryLogger runtimeLogger = null;

            // Gets a handle to the standard output and error streams.
            var stdOut = Console.Out;
            var stdErr = Console.Error;

            try
            {
                // Creates a new instance of the bug-finding runtime.
                if (this.TestRuntimeFactoryMethod != null)
                {
                    runtime = (SystematicTestingRuntime)this.TestRuntimeFactoryMethod.Invoke(
                        null,
                        new object[] { this.Configuration, this.Strategy, this.Reporter });
                }
                else
                {
                    runtime = new SystematicTestingRuntime(this.Configuration, this.Strategy, this.Reporter);
                }

                if (this.Configuration.EnableDataRaceDetection)
                {
                    this.Reporter.RegisterRuntime(runtime);
                }

                // If verbosity is turned off, then intercept the program log, and also dispose
                // the standard output and error streams.
                if (!this.Configuration.IsVerbose)
                {
                    runtimeLogger = new InMemoryLogger(true);
                    runtime.SetLogger(runtimeLogger);

                    var writer = new LogWriter(new DisposingLogger());
                    Console.SetOut(writer);
                    Console.SetError(writer);
                }

                // Runs the test inside the test-harness machine.
                if (this.TestAction != null)
                {
                    runtime.RunTestHarness(this.TestAction, this.TestName);
                }
                else
                {
                    runtime.RunTestHarness(this.TestFunction, this.TestName);
                }

                // Wait for the test to terminate.
                runtime.WaitAsync().Wait();

                // Invokes user-provided cleanup for this iteration.
                if (this.TestIterationDisposeMethod != null)
                {
                    // Disposes the test state.
                    this.TestIterationDisposeMethod.Invoke(null, null);
                }

                // Invoke the per iteration callbacks, if any.
                foreach (var callback in this.PerIterationCallbacks)
                {
                    callback(iteration);
                }

                if (this.Configuration.RaceFound)
                {
                    runtime.Scheduler.NotifyAssertionFailure("Found a race", killTasks: false, cancelExecution: false);
                    foreach (var report in this.TestReport.BugReports)
                    {
                        runtime.Logger.WriteLine(report);
                    }
                }

                // Checks that no monitor is in a hot state at termination. Only
                // checked if no safety property violations have been found.
                if (!runtime.Scheduler.BugFound)
                {
                    runtime.CheckNoMonitorInHotStateAtTermination();
                }

                if (runtime.Scheduler.BugFound)
                {
                    this.ErrorReporter.WriteErrorLine(runtime.Scheduler.BugReport);
                }

                this.GatherIterationStatistics(runtime);

                if (this.TestReport.NumOfFoundBugs > 0)
                {
                    if (runtimeLogger != null)
                    {
                        this.ReadableTrace  = runtimeLogger.ToString();
                        this.ReadableTrace += this.TestReport.GetText(this.Configuration, "<StrategyLog>");
                    }

                    this.BugTrace = runtime.BugTrace;
                    this.ConstructReproducableTrace(runtime);
                }
            }
            finally
            {
                if (!this.Configuration.IsVerbose)
                {
                    // Restores the standard output and error streams.
                    Console.SetOut(stdOut);
                    Console.SetError(stdErr);
                }

                if (this.Configuration.PerformFullExploration && runtime.Scheduler.BugFound)
                {
                    this.Logger.WriteLine($"..... Iteration #{iteration + 1} " +
                                          $"triggered bug #{this.TestReport.NumOfFoundBugs} " +
                                          $"[task-{this.Configuration.TestingProcessId}]");
                }

                // Cleans up the runtime before the next iteration starts.
                runtimeLogger?.Dispose();
                runtime?.Dispose();
            }
        }
Exemplo n.º 8
0
        /// <summary>
        /// Creates a new testing task.
        /// </summary>
        protected override Task CreateTestingTask()
        {
            return(new Task(() =>
            {
                // Runtime used to serialize and test the program.
                SystematicTestingRuntime runtime = null;

                // Logger used to intercept the program output if no custom logger
                // is installed and if verbosity is turned off.
                InMemoryLogger runtimeLogger = null;

                // Gets a handle to the standard output and error streams.
                var stdOut = Console.Out;
                var stdErr = Console.Error;

                try
                {
                    if (this.TestInitMethod != null)
                    {
                        // Initializes the test state.
                        this.TestInitMethod.Invoke(null, Array.Empty <object>());
                    }

                    // Creates a new instance of the testing runtime.
                    if (this.TestRuntimeFactoryMethod != null)
                    {
                        runtime = (SystematicTestingRuntime)this.TestRuntimeFactoryMethod.Invoke(
                            null,
                            new object[] { this.Configuration, this.Strategy, this.Reporter });
                    }
                    else
                    {
                        runtime = new SystematicTestingRuntime(this.Configuration, this.Strategy, this.Reporter);
                    }

                    // If verbosity is turned off, then intercept the program log, and also redirect
                    // the standard output and error streams into the runtime logger.
                    if (!this.Configuration.IsVerbose)
                    {
                        runtimeLogger = new InMemoryLogger();
                        runtime.SetLogger(runtimeLogger);

                        var writer = new LogWriter(new NulLogger());
                        Console.SetOut(writer);
                        Console.SetError(writer);
                    }

                    // Runs the test inside the test-harness machine.
                    if (this.TestAction != null)
                    {
                        runtime.RunTestHarness(this.TestAction, this.TestName);
                    }
                    else
                    {
                        runtime.RunTestHarness(this.TestFunction, this.TestName);
                    }

                    // Wait for the test to terminate.
                    runtime.WaitAsync().Wait();

                    // Invokes user-provided cleanup for this iteration.
                    if (this.TestIterationDisposeMethod != null)
                    {
                        // Disposes the test state.
                        this.TestIterationDisposeMethod.Invoke(null, Array.Empty <object>());
                    }

                    // Invokes user-provided cleanup for all iterations.
                    if (this.TestDisposeMethod != null)
                    {
                        // Disposes the test state.
                        this.TestDisposeMethod.Invoke(null, Array.Empty <object>());
                    }

                    this.InternalError = (this.Strategy as ReplayStrategy).ErrorText;

                    // Checks that no monitor is in a hot state at termination. Only
                    // checked if no safety property violations have been found.
                    if (!runtime.Scheduler.BugFound && this.InternalError.Length == 0)
                    {
                        runtime.CheckNoMonitorInHotStateAtTermination();
                    }

                    if (runtime.Scheduler.BugFound && this.InternalError.Length == 0)
                    {
                        this.ErrorReporter.WriteErrorLine(runtime.Scheduler.BugReport);
                    }

                    TestReport report = runtime.Scheduler.GetReport();
                    report.CoverageInfo.Merge(runtime.CoverageInfo);
                    this.TestReport.Merge(report);
                }
                catch (TargetInvocationException ex)
                {
                    if (!(ex.InnerException is TaskCanceledException))
                    {
                        ExceptionDispatchInfo.Capture(ex.InnerException).Throw();
                    }
                }
                finally
                {
                    if (!this.Configuration.IsVerbose)
                    {
                        // Restores the standard output and error streams.
                        Console.SetOut(stdOut);
                        Console.SetError(stdErr);
                    }

                    // Cleans up the runtime.
                    runtimeLogger?.Dispose();
                    runtime?.Dispose();
                }
            }, this.CancellationTokenSource.Token));
        }
Exemplo n.º 9
0
 /// <summary>
 /// Initializes a new instance of the <see cref="MockSharedRegister{T}"/> class.
 /// </summary>
 public MockSharedRegister(T value, SystematicTestingRuntime runtime)
 {
     this.Runtime         = runtime;
     this.RegisterMachine = this.Runtime.CreateMachine(typeof(SharedRegisterMachine <T>));
     this.Runtime.SendEvent(this.RegisterMachine, SharedRegisterEvent.SetEvent(value));
 }
Exemplo n.º 10
0
 /// <summary>
 /// Registers the testing runtime.
 /// </summary>
 public void RegisterRuntime(IMachineRuntime runtime)
 {
     runtime.Assert((runtime as SystematicTestingRuntime) != null,
                    "Requires passed runtime to support method GetCurrentMachineId");
     this.Runtime = runtime as SystematicTestingRuntime;
 }
Exemplo n.º 11
0
 /// <summary>
 /// Initializes a new instance of the <see cref="StateCache"/> class.
 /// </summary>
 internal StateCache(SystematicTestingRuntime runtime)
 {
     this.Runtime      = runtime;
     this.Fingerprints = new HashSet <Fingerprint>();
 }
Exemplo n.º 12
0
 /// <summary>
 /// Initializes a new instance of the <see cref="ControlledTaskScheduler"/> class.
 /// </summary>
 internal ControlledTaskScheduler(SystematicTestingRuntime runtime, ConcurrentDictionary <int, AsyncMachine> controlledTaskMap)
 {
     this.Runtime           = runtime;
     this.ControlledTaskMap = controlledTaskMap;
 }