private static void BeginRun()
{
// Find all test classes.
var entryAssembly = Assembly.GetEntryAssembly();
if (entryAssembly == null)
{
return;
}
Type[] testClasses = entryAssembly.GetTypes().AsParallel().Where(x => x.GetCustomAttributes(typeof(TestAttribute), true).Length > 0).ToArray();
// Find all test functions in these classes.
var tests = new List <MethodInfo>();
MethodInfo initMethod = null;
foreach (Type classType in testClasses)
{
// Check if filtering by tag.
var t = (TestAttribute)classType.GetCustomAttributes(typeof(TestAttribute), true).FirstOrDefault();
string tag = (t?.Tag ?? "").ToLower();
if (tag == "testinit")
{
initMethod = classType.GetMethods().FirstOrDefault();
continue;
}
if (!string.IsNullOrEmpty(TestTag) && tag != TestTag)
{
#if TEST_DEBUG
Log.Trace($"Skipping class {classType} because it doesn't match tag filter '{TestTag}'.", CustomMSource.TestRunner);
#endif
continue;
}
if (string.IsNullOrEmpty(TestTag) && (t?.TagOnly ?? false))
{
#if TEST_DEBUG
Log.Trace($"Skipping class {classType} because it will only run with the tag '{t.Tag}'.", CustomMSource.TestRunner);
#endif
continue;
}
// Find all test functions in this class.
tests.AddRange(classType.GetMethods().AsParallel().Where(x => x.GetCustomAttributes(typeof(TestAttribute), true).Length > 0).ToArray());
}
if (initMethod != null)
{
Engine.Log.Info("Executing test init method...", TestRunnerLogger.TestRunnerSrc);
Type initClassType = initMethod.DeclaringType;
Debug.Assert(initClassType != null);
object initClassInstance = Activator.CreateInstance(initClassType);
initMethod.Invoke(initClassInstance, new object[] { });
Engine.Log.Info("Tests initialized.", TestRunnerLogger.TestRunnerSrc);
}
Type currentClass = null;
object currentClassInstance = null;
float classTimer = 0;
var timeTracker = new Stopwatch();
var failedTests = 0;
foreach (MethodInfo func in tests)
{
// Create an instance of the test class.
if (currentClass != func.DeclaringType)
{
if (currentClass != null)
{
Engine.Log.Info($"Test class {currentClass} completed in {classTimer}ms!", TestRunnerLogger.TestRunnerSrc);
}
currentClass = func.DeclaringType;
if (currentClass == null)
{
throw new Exception($"Declaring type of function {func.Name} is missing.");
}
currentClassInstance = Activator.CreateInstance(currentClass);
classTimer = 0;
Engine.Log.Info($"Running test class {currentClass}...", TestRunnerLogger.TestRunnerSrc);
}
// Run test.
Engine.Log.Info($" Running test {func.Name}...", TestRunnerLogger.TestRunnerSrc);
timeTracker.Restart();
#if !THROW_EXCEPTIONS
try
{
#endif
func.Invoke(currentClassInstance, new object[] { });
// Check if errored in the loop.
if (_loopException != null)
{
var wrapped = new Exception("Exception in test engine loop.", _loopException);
_loopException = null;
throw wrapped;
}
#if !THROW_EXCEPTIONS
}
catch (ImageDerivationException)
{
failedTests++;
}
catch (Exception ex)
{
failedTests++;
if (ex.InnerException is ImageDerivationException)
{
Engine.Log.Error($"{ex.InnerException.Message}", TestRunnerLogger.TestRunnerSrc);
continue;
}
Engine.Log.Error($" Test {func.Name} failed - {ex}", TestRunnerLogger.TestRunnerSrc);
Debug.Assert(false);
}
#endif
Engine.Log.Info($" Test {func.Name} completed in {timeTracker.ElapsedMilliseconds}ms!", TestRunnerLogger.TestRunnerSrc);
classTimer += timeTracker.ElapsedMilliseconds;
}
Engine.Log.Info($"Test completed: {tests.Count - failedTests}/{tests.Count}!", TestRunnerLogger.TestRunnerSrc);
// If not the master - then nothing else to do.
if (TestRunId != RunnerId.ToString())
{
return;
}
var results = new List <string> {
$"Master: Test completed: {tests.Count - failedTests}/{tests.Count}!"
};
int totalTests = tests.Count;
var error = false;
// Wait for linked runners to exit.
foreach (LinkedRunner linked in _linkedRunners)
{
Engine.Log.Info("----------------------------------------------------------------------", TestRunnerLogger.TestRunnerSrc);
Engine.Log.Info($"Waiting for LR{linked.Id} - ({linked.Args})", TestRunnerLogger.TestRunnerSrc);
int exitCode = linked.WaitForFinish(out string output, out string errorOutput);
output = output.Trim();
errorOutput = errorOutput.Trim();
// Try to find the test completed line.
Match match = _testCompletedRegex.Match(output);
var result = "";
if (match.Success)
{
try
{
int testsSuccess = int.Parse(match.Groups[1].Value);
int testsRun = int.Parse(match.Groups[2].Value);
int failed = testsRun - testsSuccess;
failedTests += failed;
totalTests += testsRun;
result = match.Groups[0].Value;
}
catch (Exception)
{
Engine.Log.Info($"Couldn't read tests completed from LR{linked.Id}.", TestRunnerLogger.TestRunnerSrc);
}
}
else
{
result = "<Unknown>/<Unknown>";
}
var anyError = "";
if (!string.IsNullOrEmpty(errorOutput))
{
error = true;
anyError = "ERR ";
}
results.Add($"LR{linked.Id} ({linked.Args}) {result} {anyError}{linked.TimeElapsed}ms");
Engine.Log.Info($"LR{linked.Id} exited with code {exitCode}.", TestRunnerLogger.TestRunnerSrc);
Engine.Log.Info($"Dumping log from LR{linked.Id}\n{output}", TestRunnerLogger.TestRunnerSrc);
if (!string.IsNullOrEmpty(errorOutput))
{
Engine.Log.Info($"[LR{linked.Id}] Error Output\n{errorOutput}", TestRunnerLogger.TestRunnerSrc);
}
}
// Post final results.
Engine.Log.Info($"Final test results: {totalTests - failedTests}/{totalTests} {(error ? "Errors found!" : "")}!", TestRunnerLogger.TestRunnerSrc);
foreach (string r in results)
{
Engine.Log.Info($" {r}", TestRunnerLogger.TestRunnerSrc);
}
if (error || failedTests > 0)
{
Environment.Exit(1);
}
}
/// <summary>
/// Run tests.
/// </summary>
/// <param name="engineConfig">The default engine config. All configs in "otherConfigs" are modifications of this one.</param>
/// <param name="args">The execution args passed to the Main. This is needed to coordinate linked runners.</param>
/// <param name="otherConfigs">List of engine configurations to spawn runners with.</param>
/// <param name="screenResultDb">Database of screenshot results to compare against when using VerifyImage</param>
public static void RunTests(
Configurator engineConfig,
string[] args = null,
Dictionary <string, Action <Configurator> > otherConfigs = null,
Dictionary <string, byte[]> screenResultDb = null
)
{
if (args == null)
{
args = new string[] { }
}
;
_otherConfigs = otherConfigs ?? new Dictionary <string, Action <Configurator> >();
_screenResultDb = screenResultDb ?? new Dictionary <string, byte[]>();
// Check for test run id. This signifies whether the runner is linked.
TestRunId = CommandLineParser.FindArgument(args, "testRunId=", out string testRunId) ? testRunId : RunnerId.ToString().ToLower();
// Check if running only specific tests.
if (CommandLineParser.FindArgument(args, "tag=", out string testTag))
{
TestTag = testTag;
}
// Check if a custom engine config is to be loaded. This check is a bit elaborate since the config params are merged with the linked params.
string argsJoined = string.Join(" ", args);
string customConfig = (from possibleConfigs in _otherConfigs where argsJoined.Contains(possibleConfigs.Key) select possibleConfigs.Key).FirstOrDefault();
CustomConfig = customConfig;
string resultFolder = CommandLineParser.FindArgument(args, "folder=", out string folderPassed) ? folderPassed : $"{DateTime.Now:MM-dd-yyyy(HH.mm.ss)}";
TestRunFolder = Path.Join(AppDomain.CurrentDomain.BaseDirectory, "TestResults", resultFolder);
RunnerReferenceImageFolder = Path.Join(TestRunFolder, RenderResultStorage, $"Runner {TestTagDisplay}");
// Check if master runner.
bool linked = TestRunId != RunnerId.ToString();
LoggingProvider log = new TestRunnerLogger(linked, Path.Join(TestRunFolder, "Logs"));
// Set the default engine settings for the test runner.
Configurator config = engineConfig;
config.DebugMode = true;
config.LoopFactory = TestLoop;
config.Logger = log;
if (customConfig != null && _otherConfigs.ContainsKey(customConfig) && _otherConfigs[customConfig] != null)
{
CustomConfig = customConfig;
Engine.Log.Info($"Loading custom engine config - {customConfig}...", TestRunnerLogger.TestRunnerSrc);
_otherConfigs[customConfig](config);
}
// Perform light setup.
Engine.LightSetup(config);
// Run linked runners (if the master).
if (linked)
{
log.Info($"I am a linked runner with arguments {string.Join(" ", args)}", TestRunnerLogger.TestRunnerSrc);
}
else
{
// Spawn linked runners
if (!NoLinkedRunners)
{
// Spawn a runner for each runtime config.
foreach ((string arg, Action <Configurator> _) in _otherConfigs)
{
_linkedRunners.Add(new LinkedRunner(arg));
}
}
}
// Check if running tests without an engine instance - this shouldn't be used with a tag because most tests except an instance.
if (CommandLineParser.FindArgument(args, "testOnly", out string _))
{
Task tests = Task.Run(BeginRun);
while (!tests.IsCompleted)
{
TestLoopUpdate();
}
Engine.Quit();
return;
}
// Perform engine setup.
Engine.Setup(config);
if (Engine.Renderer == null)
{
return;
}
// Move the camera center in a way that its center is 0,0
Engine.Renderer.Camera.Position += new Vector3(Engine.Renderer.Camera.WorldToScreen(Vector2.Zero), 0);
Task.Run(() =>
{
// Wait for the engine to start.
while (Engine.Status != EngineStatus.Running)
{
}
// If crashed.
if (Engine.Status == EngineStatus.Stopped)
{
return;
}
// Name the thread.
if (Engine.Host?.NamedThreads ?? false)
{
Thread.CurrentThread.Name ??= "Runner Thread";
}
BeginRun();
Engine.Quit();
// Wait for the engine to stop.
while (Engine.Status == EngineStatus.Running)
{
}
});
Engine.Run();
}