private void TestWithErrors(Delegate test, Configuration configuration, TestErrorChecker errorChecker, bool replay)
{
configuration ??= GetConfiguration();
ILogger logger = this.GetLogger(configuration);
try
{
var engine = RunTest(test, configuration, logger);
CheckErrors(engine, errorChecker);
if (replay)
{
configuration.WithReplayStrategy(engine.ReproducibleTrace);
engine = RunTest(test, configuration, logger);
string replayError = engine.SchedulingContext.GetReplayError();
Assert.True(replayError.Length is 0, replayError);
CheckErrors(engine, errorChecker);
}
}
catch (Exception ex)
{
Assert.False(true, ex.Message + "\n" + ex.StackTrace);
}
finally
{
logger.Dispose();
}
}
private static void CheckErrors(TestingEngine engine, TestErrorChecker errorChecker)
{
Assert.True(engine.TestReport.NumOfFoundBugs > 0, "Expected bugs to be found, but we found none");
foreach (var bugReport in engine.TestReport.BugReports)
{
errorChecker(bugReport);
}
}
protected void TestWithError(Func <IActorRuntime, Task> test, TestErrorChecker errorChecker, Configuration configuration = null,
bool replay = false)
{
if (this.SchedulingPolicy is SchedulingPolicy.None)
{
this.RunWithErrorsAsync(test, configuration, errorChecker).Wait();
}
else
{
this.TestWithErrors(test, configuration, errorChecker, replay);
}
}
protected void TestWithError(Action test, TestErrorChecker errorChecker, Configuration configuration = null,
bool replay = false)
{
if (this.SchedulingPolicy is SchedulingPolicy.None)
{
this.RunWithErrors((r) => test(), configuration, errorChecker);
}
else
{
this.TestWithErrors(test, configuration, errorChecker, replay);
}
}
protected void TestWithError(Func <IActorRuntime, Task> test, TestErrorChecker errorChecker, Configuration configuration = null,
bool replay = false)
{
if (this.IsSystematicTest)
{
this.TestWithErrors(test, configuration, errorChecker, replay);
}
else
{
this.RunWithErrorsAsync(test, configuration, errorChecker).Wait();
}
}
protected void TestWithError(Action test, TestErrorChecker errorChecker, Configuration configuration = null,
bool replay = false)
{
if (this.IsSystematicTest)
{
this.TestWithErrors(test, configuration, errorChecker, replay);
}
else
{
this.RunWithErrors((r) => test(), configuration, errorChecker);
}
}
protected void TestWithError(Func <Task> test, TestErrorChecker errorChecker, Configuration configuration = null,
bool replay = false)
{
if (this.SystematicTest)
{
this.TestWithErrors(test, configuration, errorChecker, replay);
}
else
{
this.RunWithErrorsAsync(async(r) => await test(), configuration, errorChecker).Wait();
}
}
private void TestWithErrors(Delegate test, Configuration configuration, TestErrorChecker errorChecker, bool replay)
{
configuration = configuration ?? GetConfiguration();
TextWriter logger;
if (configuration.IsVerbose)
{
logger = new TestOutputLogger(this.TestOutput, true);
}
else
{
logger = TextWriter.Null;
}
try
{
var engine = RunTest(test, configuration, logger);
CheckErrors(engine, errorChecker);
if (replay)
{
configuration.WithReplayStrategy(engine.ReproducableTrace);
engine = RunTest(test, configuration, logger);
string replayError = (engine.Strategy as ReplayStrategy).ErrorText;
Assert.True(replayError.Length == 0, replayError);
CheckErrors(engine, errorChecker);
}
}
catch (Exception ex)
{
Assert.False(true, ex.Message + "\n" + ex.StackTrace);
}
finally
{
logger.Dispose();
}
}
private void TestWithErrors(Delegate test, Configuration configuration, TestErrorChecker errorChecker, bool replay)
{
configuration ??= this.GetConfiguration();
if (this.SchedulingPolicy is SchedulingPolicy.Fuzzing)
{
// Increase iterations during fuzzing as some bugs might be harder to be found.
configuration = configuration.WithTestingIterations(configuration.TestingIterations * 20);
}
ILogger logger = this.GetLogger(configuration);
try
{
var engine = RunTest(test, configuration, logger);
CheckErrors(engine, errorChecker);
if (replay && this.SchedulingPolicy is SchedulingPolicy.Systematic)
{
configuration.WithReplayStrategy(engine.ReproducibleTrace);
engine = RunTest(test, configuration, logger);
string replayError = engine.Scheduler.GetReplayError();
Assert.True(replayError.Length is 0, replayError);
CheckErrors(engine, errorChecker);
}
}
catch (Exception ex)
{
Assert.False(true, ex.Message + "\n" + ex.StackTrace);
}
finally
{
logger.Dispose();
}
}
private async Task RunWithErrorsAsync(Func <IActorRuntime, Task> test, Configuration configuration, TestErrorChecker errorChecker)
{
configuration ??= this.GetConfiguration();
string errorMessage = string.Empty;
ILogger logger = this.GetLogger(configuration);
try
{
configuration.IsMonitoringEnabledInInProduction = true;
var runtime = ActorRuntimeFactory.Create(configuration);
var errorCompletion = new TaskCompletionSource <Exception>();
runtime.OnFailure += (e) =>
{
errorCompletion.TrySetResult(e);
};
runtime.Logger = logger;
for (int i = 0; i < configuration.TestingIterations; i++)
{
await test(runtime);
if (configuration.TestingIterations is 1)
{
Assert.True(errorCompletion.Task.Wait(GetExceptionTimeout()), "Timeout waiting for error");
errorMessage = ExtractErrorMessage(errorCompletion.Task.Result);
}
}
}
catch (Exception ex)
{
errorMessage = ExtractErrorMessage(ex);
}
finally
{
logger.Dispose();
}
if (string.IsNullOrEmpty(errorMessage))
{
Assert.True(false, string.Format("Error not found after all {0} test iterations", configuration.TestingIterations));
}
errorChecker(errorMessage);
}
private async Task RunWithErrorsAsync(Func <IActorRuntime, Task> test, Configuration configuration, TestErrorChecker errorChecker)
{
configuration = configuration ?? GetConfiguration();
TextWriter logger;
if (configuration.IsVerbose)
{
logger = new TestOutputLogger(this.TestOutput, true);
}
else
{
logger = TextWriter.Null;
}
try
{
var runtime = RuntimeFactory.Create(configuration);
runtime.SetLogger(logger);
for (int i = 0; i < configuration.TestingIterations; i++)
{
await test(runtime);
}
}
catch (Exception ex)
{
errorChecker(GetFirstLine(ex.Message));
}
finally
{
logger.Dispose();
}
}
private void RunWithErrors(Action <IActorRuntime> test, Configuration configuration, TestErrorChecker errorChecker)
{
configuration = configuration ?? GetConfiguration();
TextWriter logger;
if (configuration.IsVerbose)
{
logger = new TestOutputLogger(this.TestOutput, true);
}
else
{
logger = TextWriter.Null;
}
try
{
var runtime = RuntimeFactory.Create(configuration);
runtime.SetLogger(logger);
for (int i = 0; i < configuration.TestingIterations; i++)
{
test(runtime);
}
}
catch (Exception ex)
{
var msg = ex.Message;
if (ex is AggregateException ae)
{
StringBuilder sb = new StringBuilder();
foreach (var e in ae.InnerExceptions)
{
sb.AppendLine(e.Message);
}
msg = sb.ToString();
}
errorChecker(msg);
}
finally
{
logger.Dispose();
}
}
