protected void TestWithError(Action <IMachineRuntime> test, Configuration configuration = null,
string[] expectedErrors = null, bool replay = false)
{
configuration = configuration ?? GetConfiguration();
var logger = new Common.TestOutputLogger(this.TestOutput);
try
{
var bfEngine = BugFindingEngine.Create(configuration, test);
bfEngine.SetLogger(logger);
bfEngine.Run();
CheckErrors(bfEngine, expectedErrors);
if (replay && !configuration.EnableCycleDetection)
{
var rEngine = ReplayEngine.Create(configuration, test, bfEngine.ReproducableTrace);
rEngine.SetLogger(logger);
rEngine.Run();
Assert.True(rEngine.InternalError.Length == 0, rEngine.InternalError);
CheckErrors(rEngine, expectedErrors);
}
}
catch (Exception ex)
{
Assert.False(true, ex.Message + "\n" + ex.StackTrace);
}
finally
{
logger.Dispose();
}
}
protected ITestingEngine AssertSucceeded(Configuration configuration, Action <PSharpRuntime> test)
{
InMemoryLogger logger = new InMemoryLogger();
BugFindingEngine engine = null;
try
{
engine = BugFindingEngine.Create(configuration, test);
engine.SetLogger(logger);
engine.Run();
var numErrors = engine.TestReport.NumOfFoundBugs;
Assert.True(numErrors == 0, GetBugReport(engine));
}
catch (Exception ex)
{
Assert.False(true, ex.Message + "\n" + ex.StackTrace);
}
finally
{
logger.Dispose();
}
return(engine);
}
protected ITestingEngine Test(Func <IMachineRuntime, Task> test, Configuration configuration = null)
{
configuration = configuration ?? GetConfiguration();
BugFindingEngine engine = BugFindingEngine.Create(configuration, test);
return(this.Test(engine));
}
protected void AssertFailed(Configuration configuration, Action <PSharpRuntime> test, int numExpectedErrors, ISet <string> expectedOutputs)
{
InMemoryLogger logger = new InMemoryLogger();
try
{
var bfEngine = BugFindingEngine.Create(configuration, test);
bfEngine.SetLogger(logger);
bfEngine.Run();
CheckErrors(bfEngine, numExpectedErrors, expectedOutputs);
if (!configuration.EnableCycleDetection)
{
var rEngine = ReplayEngine.Create(configuration, test, bfEngine.ReproducableTrace);
rEngine.SetLogger(logger);
rEngine.Run();
Assert.True(rEngine.InternalError.Length == 0, rEngine.InternalError);
CheckErrors(rEngine, numExpectedErrors, expectedOutputs);
}
}
catch (Exception ex)
{
Assert.False(true, ex.Message + "\n" + ex.StackTrace);
}
finally
{
logger.Dispose();
}
}
protected void AssertFailedWithException(Configuration configuration, Action <IMachineRuntime> test, Type exceptionType)
{
Assert.True(exceptionType.IsSubclassOf(typeof(Exception)), "Please configure the test correctly. " +
$"Type '{exceptionType}' is not an exception type.");
var logger = new Common.TestOutputLogger(this.TestOutput);
try
{
var bfEngine = BugFindingEngine.Create(configuration, test);
bfEngine.SetLogger(logger);
bfEngine.Run();
CheckErrors(bfEngine, exceptionType);
if (!configuration.EnableCycleDetection)
{
var rEngine = ReplayEngine.Create(configuration, test, bfEngine.ReproducableTrace);
rEngine.SetLogger(logger);
rEngine.Run();
Assert.True(rEngine.InternalError.Length == 0, rEngine.InternalError);
CheckErrors(rEngine, exceptionType);
}
}
catch (Exception ex)
{
Assert.False(true, ex.Message + "\n" + ex.StackTrace);
}
finally
{
logger.Dispose();
}
}
protected void AssertFailedWithException(Configuration configuration, Action <PSharpRuntime> test, Type exceptionType)
{
Assert.True(exceptionType.IsSubclassOf(typeof(Exception)), "Please configure the test correctly. " +
$"Type '{exceptionType}' is not an exception type.");
InMemoryLogger logger = new InMemoryLogger();
try
{
var engine = BugFindingEngine.Create(configuration, test);
engine.SetLogger(logger);
engine.Run();
var numErrors = engine.TestReport.NumOfFoundBugs;
Assert.Equal(1, numErrors);
var exception = this.RemoveNonDeterministicValuesFromReport(engine.TestReport.BugReports.First()).
Split(new[] { '\r', '\n' }).FirstOrDefault();
Assert.Contains("'" + exceptionType.ToString() + "'", exception);
}
catch (Exception ex)
{
Assert.False(true, ex.Message);
}
finally
{
logger.Dispose();
}
}
public void TestCustomLogWriter()
{
Configuration configuration = GetConfiguration().WithStrategy(SchedulingStrategy.DFS);
BugFindingEngine engine = BugFindingEngine.Create(configuration,
r =>
{
r.SetLogWriter(new CustomLogWriter());
r.CreateMachine(typeof(M));
});
try
{
engine.Run();
var numErrors = engine.TestReport.NumOfFoundBugs;
Assert.True(numErrors == 1, GetBugReport(engine));
Assert.True(engine.ReadableTrace != null, "Readable trace is null.");
Assert.True(engine.ReadableTrace.Length > 0, "Readable trace is empty.");
string expected = @"<TestHarnessLog> Running anonymous test.
<CreateLog>.
<StateLog>.
<ActionLog> Machine 'Microsoft.PSharp.TestingServices.Tests.LogMessages.M()' in state 'Init' invoked action 'InitOnEntry'.
<CreateLog>.
<StateLog>.
<DequeueLog> Machine 'Microsoft.PSharp.TestingServices.Tests.LogMessages.N()' in state 'Init' dequeued event 'Microsoft.PSharp.TestingServices.Tests.LogMessages.E'.
<ActionLog> Machine 'Microsoft.PSharp.TestingServices.Tests.LogMessages.N()' in state 'Init' invoked action 'Act'.
<DequeueLog> Machine 'Microsoft.PSharp.TestingServices.Tests.LogMessages.M()' in state 'Init' dequeued event 'Microsoft.PSharp.TestingServices.Tests.LogMessages.E'.
<ActionLog> Machine 'Microsoft.PSharp.TestingServices.Tests.LogMessages.M()' in state 'Init' invoked action 'Act'.
<ErrorLog> Bug found!
<StrategyLog> Found bug using 'DFS' strategy.
<StrategyLog> Testing statistics:
<StrategyLog> Found bug.
<StrategyLog> Scheduling statistics:
<StrategyLog> Explored schedule: fair and unfair.
<StrategyLog> Found .% buggy schedules.";
string actual = Regex.Replace(engine.ReadableTrace.ToString(), "[0-9]", string.Empty);
HashSet<string> expectedSet = new HashSet<string>(Regex.Split(expected, "\r\n|\r|\n"));
HashSet<string> actualSet = new HashSet<string>(Regex.Split(actual, "\r\n|\r|\n"));
Assert.Equal(expected, actual);
}
catch (Exception ex)
{
Assert.False(true, ex.Message + "\n" + ex.StackTrace);
}
}
protected void AssertSucceeded(Configuration configuration, Action <PSharpRuntime> test)
{
InMemoryLogger logger = new InMemoryLogger();
try
{
var engine = BugFindingEngine.Create(configuration, test);
engine.SetLogger(logger);
engine.Run();
var numErrors = engine.TestReport.NumOfFoundBugs;
Assert.Equal(0, numErrors);
}
catch (Exception ex)
{
Assert.False(true, ex.Message);
}
finally
{
logger.Dispose();
}
}
protected void AssertSucceeded(Configuration configuration, Action <IMachineRuntime> test)
{
var logger = new Common.TestOutputLogger(this.TestOutput);
try
{
var engine = BugFindingEngine.Create(configuration, test);
engine.SetLogger(logger);
engine.Run();
var numErrors = engine.TestReport.NumOfFoundBugs;
Assert.True(numErrors == 0, GetBugReport(engine));
}
catch (Exception ex)
{
Assert.False(true, ex.Message + "\n" + ex.StackTrace);
}
finally
{
logger.Dispose();
}
}
protected void AssertFailed(Configuration configuration, Action <PSharpRuntime> test, int numExpectedErrors, ISet <string> expectedOutputs)
{
InMemoryLogger logger = new InMemoryLogger();
try
{
var engine = BugFindingEngine.Create(configuration, test);
engine.SetLogger(logger);
engine.Run();
var numErrors = engine.TestReport.NumOfFoundBugs;
Assert.Equal(numExpectedErrors, numErrors);
if (expectedOutputs.Count > 0)
{
var bugReports = new HashSet <string>();
foreach (var bugReport in engine.TestReport.BugReports)
{
var actual = this.RemoveNonDeterministicValuesFromReport(bugReport);
bugReports.Add(actual);
}
foreach (var expected in expectedOutputs)
{
Assert.Contains(expected, bugReports);
}
}
}
catch (Exception ex)
{
Assert.False(true, ex.Message);
}
finally
{
logger.Dispose();
}
}
