public void GetTestResults_OutputWithSkippedTestAsLastTest_AllResultsAreFound() { var cases = new List <TestCase> { TestDataCreator.ToTestCase("Test.Succeed", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"), TestDataCreator.ToTestCase("Test.Skip", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"), TestDataCreator.ToTestCase("Test.Fail", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"), }; var results = new StandardOutputTestResultParser(cases, ConsoleOutputWithSkippedTestAsLastTest, TestEnvironment.Logger).GetTestResults(); results.Should().HaveCount(3); var result = results[0]; result.TestCase.FullyQualifiedName.Should().Be("Test.Succeed"); XmlTestResultParserTests.AssertTestResultIsPassed(result); result = results[1]; result.TestCase.FullyQualifiedName.Should().Be("Test.Fail"); XmlTestResultParserTests.AssertTestResultIsFailure(result); result = results[2]; result.TestCase.FullyQualifiedName.Should().Be("Test.Skip"); XmlTestResultParserTests.AssertTestResultIsSkipped(result); }
private void ReportTestStart(string line) { string qualifiedTestname = StandardOutputTestResultParser.RemovePrefix(line).Trim(); TestCase testCase = StandardOutputTestResultParser.FindTestcase(qualifiedTestname, _testCasesRun); _reporter.ReportTestsStarted(testCase.Yield()); }
private void CheckStandardOutputResultParser(IEnumerable <TestCase> testCasesRun, IEnumerable <string> consoleOutput, IList <TestResult> results, TestCase crashedTestCase) { var parser = new StandardOutputTestResultParser(testCasesRun, consoleOutput, MockLogger.Object); parser.GetTestResults().Should().BeEquivalentTo(results); parser.CrashedTestCase.Should().Be(crashedTestCase); }
private List <TestResult> ComputeTestResults(List <string> consoleOutput) { IList <TestCase> cases = new List <TestCase>(); cases.Add(TestDataCreator.ToTestCase("TestMath.AddFails", TestDataCreator.DummyExecutable, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp")); cases.Add(TestDataCreator.ToTestCase("TestMath.Crash", TestDataCreator.DummyExecutable, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp")); cases.Add(TestDataCreator.ToTestCase("TestMath.AddPasses", TestDataCreator.DummyExecutable, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp")); StandardOutputTestResultParser parser = new StandardOutputTestResultParser(cases, consoleOutput, TestEnvironment, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\"); return(parser.GetTestResults()); }
public void ReportLine(string line) { if (StandardOutputTestResultParser.IsRunLine(line)) { if (_consoleOutput.Count > 0) { ReportTestResult(); } ReportTestStart(line); } _consoleOutput.Add(line); }
public void GetTestResults_OutputWithPrefixingTest_BothTestsAreFound() { var cases = new List <TestCase> { TestDataCreator.ToTestCase("Test.AB", TestDataCreator.DummyExecutable, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"), TestDataCreator.ToTestCase("Test.A", TestDataCreator.DummyExecutable, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp") }; var results = new StandardOutputTestResultParser(cases, ConsoleOutputWithPrefixingTest, TestEnvironment.Logger) .GetTestResults(); results.Count.Should().Be(2); results[0].TestCase.FullyQualifiedName.Should().Be("Test.AB"); XmlTestResultParserTests.AssertTestResultIsPassed(results[0]); results[1].TestCase.FullyQualifiedName.Should().Be("Test.A"); XmlTestResultParserTests.AssertTestResultIsPassed(results[1]); }
private TimeSpan ParseDuration(string durationInSeconds) { return (StandardOutputTestResultParser .NormalizeDuration(TimeSpan.FromSeconds(double.Parse(durationInSeconds, NumberFormatInfo)))); }
private List<TestResult> ComputeTestResults(List<string> consoleOutput) { IList<TestCase> cases = new List<TestCase>(); cases.Add(TestDataCreator.ToTestCase("TestMath.AddFails", TestDataCreator.DummyExecutable, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp")); cases.Add(TestDataCreator.ToTestCase("TestMath.Crash", TestDataCreator.DummyExecutable, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp")); cases.Add(TestDataCreator.ToTestCase("TestMath.AddPasses", TestDataCreator.DummyExecutable, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp")); StandardOutputTestResultParser parser = new StandardOutputTestResultParser(cases, consoleOutput, TestEnvironment, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\"); return parser.GetTestResults(); }
private TestResult CreateTestResult() { int currentLineIndex = 0; while (currentLineIndex < _consoleOutput.Count && !StandardOutputTestResultParser.IsRunLine(_consoleOutput[currentLineIndex])) { currentLineIndex++; } if (currentLineIndex >= _consoleOutput.Count) { return(null); } string line = _consoleOutput[currentLineIndex++]; string qualifiedTestname = StandardOutputTestResultParser.RemovePrefix(line).Trim(); TestCase testCase = StandardOutputTestResultParser.FindTestcase(qualifiedTestname, _testCasesRun); if (currentLineIndex >= _consoleOutput.Count) { CrashedTestCase = testCase; return(StandardOutputTestResultParser.CreateFailedTestResult( testCase, TimeSpan.FromMilliseconds(0), StandardOutputTestResultParser.CrashText, "")); } line = _consoleOutput[currentLineIndex++]; string errorMsg = ""; while ( !(StandardOutputTestResultParser.IsFailedLine(line) || StandardOutputTestResultParser.IsPassedLine(line)) && currentLineIndex <= _consoleOutput.Count) { errorMsg += line + "\n"; line = currentLineIndex < _consoleOutput.Count ? _consoleOutput[currentLineIndex] : ""; currentLineIndex++; } if (StandardOutputTestResultParser.IsFailedLine(line)) { ErrorMessageParser parser = new ErrorMessageParser(errorMsg, _baseDir); parser.Parse(); return(StandardOutputTestResultParser.CreateFailedTestResult( testCase, StandardOutputTestResultParser.ParseDuration(line, _logger), parser.ErrorMessage, parser.ErrorStackTrace)); } if (StandardOutputTestResultParser.IsPassedLine(line)) { return(StandardOutputTestResultParser.CreatePassedTestResult( testCase, StandardOutputTestResultParser.ParseDuration(line, _logger))); } CrashedTestCase = testCase; string message = StandardOutputTestResultParser.CrashText; message += errorMsg == "" ? "" : $"\nTest output:\n\n{errorMsg}"; TestResult result = StandardOutputTestResultParser.CreateFailedTestResult( testCase, TimeSpan.FromMilliseconds(0), message, ""); return(result); }
private List<TestResult> CollectTestResults(IEnumerable<TestCase> testCasesRun, string resultXmlFile, List<string> consoleOutput, string baseDir) { var testResults = new List<TestResult>(); TestCase[] testCasesRunAsArray = testCasesRun as TestCase[] ?? testCasesRun.ToArray(); var xmlParser = new XmlTestResultParser(testCasesRunAsArray, resultXmlFile, _testEnvironment, baseDir); var consoleParser = new StandardOutputTestResultParser(testCasesRunAsArray, consoleOutput, _testEnvironment, baseDir); testResults.AddRange(xmlParser.GetTestResults()); _testEnvironment.DebugInfo($"Collected {testResults.Count} test results from XML result file '{resultXmlFile}'"); if (testResults.Count < testCasesRunAsArray.Length) { List<TestResult> consoleResults = consoleParser.GetTestResults(); int nrOfCollectedTestResults = 0; // ReSharper disable once AccessToModifiedClosure foreach (TestResult testResult in consoleResults.Where(tr => !testResults.Exists(tr2 => tr.TestCase.FullyQualifiedName == tr2.TestCase.FullyQualifiedName))) { testResults.Add(testResult); nrOfCollectedTestResults++; } _testEnvironment.DebugInfo($"Collected {nrOfCollectedTestResults} test results from console output"); } if (testResults.Count < testCasesRunAsArray.Length) { string errorMessage, errorStackTrace = null; if (consoleParser.CrashedTestCase == null) { errorMessage = ""; } else { errorMessage = $"reason is probably a crash of test {consoleParser.CrashedTestCase.DisplayName}"; errorStackTrace = ErrorMessageParser.CreateStackTraceEntry("crash suspect", consoleParser.CrashedTestCase.CodeFilePath, consoleParser.CrashedTestCase.LineNumber.ToString()); } int nrOfCreatedTestResults = 0; // ReSharper disable once AccessToModifiedClosure foreach (TestCase testCase in testCasesRunAsArray.Where(tc => !testResults.Exists(tr => tr.TestCase.FullyQualifiedName == tc.FullyQualifiedName))) { testResults.Add(new TestResult(testCase) { ComputerName = Environment.MachineName, Outcome = TestOutcome.Skipped, ErrorMessage = errorMessage, ErrorStackTrace = errorStackTrace }); nrOfCreatedTestResults++; } _testEnvironment.DebugInfo($"Created {nrOfCreatedTestResults} test results for tests which were neither found in result XML file nor in console output"); } testResults = testResults.OrderBy(tr => tr.TestCase.FullyQualifiedName).ToList(); return testResults; }