/// <summary> /// Runs the test of an given executable. This implies to run the discover for these sources. /// </summary> /// <param name="sources">The full qualified name of an executable to run</param> /// <param name="runContext">Test context</param> /// <param name="frameworkHandle">Test frame work handle</param> public void RunTests(IEnumerable <string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { // Load settings from the context. var settings = CatchSettingsProvider.LoadSettings(runContext.RunSettings); frameworkHandle.SendMessage(TestMessageLevel.Informational, "CatchAdapter::RunTests... "); // Run tests in all included executables. foreach (var exeName in sources.Where(name => settings.IncludeTestExe(name))) { // Wrap execution in try to stop one executable's exceptions from stopping the others from being run. try { frameworkHandle.SendMessage(TestMessageLevel.Informational, "RunTest of source " + exeName); var tests = TestDiscoverer.CreateTestCases(exeName, runContext.SolutionDirectory); RunTests(tests, runContext, frameworkHandle); } catch (Exception ex) { frameworkHandle.SendMessage(TestMessageLevel.Error, "Exception running tests: " + ex.Message); frameworkHandle.SendMessage(TestMessageLevel.Error, "Exception stack: " + ex.StackTrace); } } }
/// <summary> /// Constructs a test result (messages and stack trace) from the given Expressions. /// </summary> /// <param name="Expresions">Catch expressions</param> /// <param name="testResult">Test result of the expressions</param> /// <returns>Count of failed expressions</returns> private int ConstructResult(Tests.Expression[] Expresions, TestResult testResult) { int i = 0; foreach (var expression in Expresions.Where(expr => expr.Success == "false")) { ++i; string expanded = expression.Expanded.Trim(); string original = expression.Original.Trim(); string type = expression.Type; var FilePath = TestDiscoverer.ResolvePath(expression.Filename, SolutionDirectory); var LineNumber = Int32.Parse(expression.Line); testResult.ErrorMessage += $"#{i} - {type}({original}) with expansion: ({expanded}){Environment.NewLine}"; testResult.ErrorStackTrace += $"at #{i} - {testResult.DisplayName}() in {FilePath}:line {LineNumber}{Environment.NewLine}"; } return(i); }
/// <summary> /// Creates the TestResult containing sub result, too. /// </summary> /// <param name="element">Element of the Catch test case.</param> /// <param name="testCase">Test case from framework.</param> /// <param name="name">Name of the Catch test including section names</param> private void CreateResult(Tests.TestCase element, TestCase testCase, string name) { name += element.Name; var subResult = new TestResult(testCase) { DisplayName = name.Replace(".", "\n\t"), ErrorMessage = $"", ErrorStackTrace = "", Outcome = TestOutcome.None }; if (element.Result != null) { subResult.Outcome = element.Result.Success == "true" ? TestOutcome.Passed : TestOutcome.Failed; subResult.Duration = TimeSpan.FromSeconds(Double.Parse(element.Result.Duration, CultureInfo.InvariantCulture)); } int failedExpressions = ConstructResult(element.Expressions, subResult); // Make sure the outcome is failed if the expressions have failed. if (failedExpressions != 0) { subResult.Outcome = TestOutcome.Failed; } foreach (var s in (element.Warning ?? new string[] { })) { var Expression = $"WARN: {s.Trim()}{ Environment.NewLine }"; subResult.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, Expression)); } foreach (var s in (element.Info ?? new string[] { })) { var Expression = $"INFO: {s.Trim()}{ Environment.NewLine }"; subResult.Messages.Add(new TestResultMessage(TestResultMessage.StandardOutCategory, Expression)); } // Check if this element is a failure generated by FAIL(). if (element.Failure != null) { ++failedExpressions; var LineNumber = Int32.Parse(element.Failure.Line); var FilePath = TestDiscoverer.ResolvePath(element.Failure.Filename, SolutionDirectory); subResult.ErrorMessage += $"#{failedExpressions} - FAIL({element.Failure.text.Trim()}){Environment.NewLine}"; subResult.ErrorStackTrace += $"at #{failedExpressions} - {name}() in {FilePath}:line {LineNumber}{Environment.NewLine}"; } if (subResult.Outcome == TestOutcome.None) { // Check if the OverallResults is set with an outcome. if (element.Results != null) { if (Int32.Parse(element.Results.Failures) != 0) { subResult.Outcome = TestOutcome.Failed; } else { subResult.Outcome = TestOutcome.Passed; } } } results.Add(subResult); // Try to find the failure from a subsection of this element. foreach (var section in element.Sections) { // Try to find a failure in this section. CreateResult(section, testCase, name + "."); } }