Exemplo n.º 1
0
        public void GetTestResults_OutputWithSkippedTestAsLastTest_AllResultsAreFound()
        {
            var cases = new List <TestCase>
            {
                TestDataCreator.ToTestCase("Test.Succeed", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
                TestDataCreator.ToTestCase("Test.Skip", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
                TestDataCreator.ToTestCase("Test.Fail", TestDataCreator.DummyExecutable, @"c:\somepath\source.cpp"),
            };

            var results = new StandardOutputTestResultParser(cases, ConsoleOutputWithSkippedTestAsLastTest, TestEnvironment.Logger).GetTestResults();

            results.Should().HaveCount(3);

            var result = results[0];

            result.TestCase.FullyQualifiedName.Should().Be("Test.Succeed");
            XmlTestResultParserTests.AssertTestResultIsPassed(result);

            result = results[1];
            result.TestCase.FullyQualifiedName.Should().Be("Test.Fail");
            XmlTestResultParserTests.AssertTestResultIsFailure(result);

            result = results[2];
            result.TestCase.FullyQualifiedName.Should().Be("Test.Skip");
            XmlTestResultParserTests.AssertTestResultIsSkipped(result);
        }
        private void ReportTestStart(string line)
        {
            string   qualifiedTestname = StandardOutputTestResultParser.RemovePrefix(line).Trim();
            TestCase testCase          = StandardOutputTestResultParser.FindTestcase(qualifiedTestname, _testCasesRun);

            _reporter.ReportTestsStarted(testCase.Yield());
        }
        private void CheckStandardOutputResultParser(IEnumerable <TestCase> testCasesRun, IEnumerable <string> consoleOutput,
                                                     IList <TestResult> results, TestCase crashedTestCase)
        {
            var parser = new StandardOutputTestResultParser(testCasesRun, consoleOutput, MockLogger.Object);

            parser.GetTestResults().Should().BeEquivalentTo(results);
            parser.CrashedTestCase.Should().Be(crashedTestCase);
        }
Exemplo n.º 4
0
        private List <TestResult> ComputeTestResults(List <string> consoleOutput)
        {
            IList <TestCase> cases = new List <TestCase>();

            cases.Add(TestDataCreator.ToTestCase("TestMath.AddFails", TestDataCreator.DummyExecutable, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"));
            cases.Add(TestDataCreator.ToTestCase("TestMath.Crash", TestDataCreator.DummyExecutable, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"));
            cases.Add(TestDataCreator.ToTestCase("TestMath.AddPasses", TestDataCreator.DummyExecutable, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"));
            StandardOutputTestResultParser parser = new StandardOutputTestResultParser(cases, consoleOutput, TestEnvironment, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\");

            return(parser.GetTestResults());
        }
 public void ReportLine(string line)
 {
     if (StandardOutputTestResultParser.IsRunLine(line))
     {
         if (_consoleOutput.Count > 0)
         {
             ReportTestResult();
         }
         ReportTestStart(line);
     }
     _consoleOutput.Add(line);
 }
Exemplo n.º 6
0
        public void GetTestResults_OutputWithPrefixingTest_BothTestsAreFound()
        {
            var cases = new List <TestCase>
            {
                TestDataCreator.ToTestCase("Test.AB", TestDataCreator.DummyExecutable,
                                           @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"),
                TestDataCreator.ToTestCase("Test.A", TestDataCreator.DummyExecutable,
                                           @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp")
            };

            var results = new StandardOutputTestResultParser(cases, ConsoleOutputWithPrefixingTest, TestEnvironment.Logger)
                          .GetTestResults();

            results.Count.Should().Be(2);
            results[0].TestCase.FullyQualifiedName.Should().Be("Test.AB");
            XmlTestResultParserTests.AssertTestResultIsPassed(results[0]);
            results[1].TestCase.FullyQualifiedName.Should().Be("Test.A");
            XmlTestResultParserTests.AssertTestResultIsPassed(results[1]);
        }
 private TimeSpan ParseDuration(string durationInSeconds)
 {
     return
         (StandardOutputTestResultParser
          .NormalizeDuration(TimeSpan.FromSeconds(double.Parse(durationInSeconds, NumberFormatInfo))));
 }
 private List<TestResult> ComputeTestResults(List<string> consoleOutput)
 {
     IList<TestCase> cases = new List<TestCase>();
     cases.Add(TestDataCreator.ToTestCase("TestMath.AddFails", TestDataCreator.DummyExecutable, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"));
     cases.Add(TestDataCreator.ToTestCase("TestMath.Crash", TestDataCreator.DummyExecutable, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"));
     cases.Add(TestDataCreator.ToTestCase("TestMath.AddPasses", TestDataCreator.DummyExecutable, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\consoleapplication1tests\source.cpp"));
     StandardOutputTestResultParser parser = new StandardOutputTestResultParser(cases, consoleOutput, TestEnvironment, @"c:\users\chris\documents\visual studio 2015\projects\consoleapplication1\");
     return parser.GetTestResults();
 }
        private TestResult CreateTestResult()
        {
            int currentLineIndex = 0;

            while (currentLineIndex < _consoleOutput.Count &&
                   !StandardOutputTestResultParser.IsRunLine(_consoleOutput[currentLineIndex]))
            {
                currentLineIndex++;
            }
            if (currentLineIndex >= _consoleOutput.Count)
            {
                return(null);
            }

            string   line = _consoleOutput[currentLineIndex++];
            string   qualifiedTestname = StandardOutputTestResultParser.RemovePrefix(line).Trim();
            TestCase testCase          = StandardOutputTestResultParser.FindTestcase(qualifiedTestname, _testCasesRun);

            if (currentLineIndex >= _consoleOutput.Count)
            {
                CrashedTestCase = testCase;
                return(StandardOutputTestResultParser.CreateFailedTestResult(
                           testCase,
                           TimeSpan.FromMilliseconds(0),
                           StandardOutputTestResultParser.CrashText,
                           ""));
            }

            line = _consoleOutput[currentLineIndex++];

            string errorMsg = "";

            while (
                !(StandardOutputTestResultParser.IsFailedLine(line) ||
                  StandardOutputTestResultParser.IsPassedLine(line)) &&
                currentLineIndex <= _consoleOutput.Count)
            {
                errorMsg += line + "\n";
                line      = currentLineIndex < _consoleOutput.Count ? _consoleOutput[currentLineIndex] : "";
                currentLineIndex++;
            }
            if (StandardOutputTestResultParser.IsFailedLine(line))
            {
                ErrorMessageParser parser = new ErrorMessageParser(errorMsg, _baseDir);
                parser.Parse();
                return(StandardOutputTestResultParser.CreateFailedTestResult(
                           testCase,
                           StandardOutputTestResultParser.ParseDuration(line, _logger),
                           parser.ErrorMessage,
                           parser.ErrorStackTrace));
            }
            if (StandardOutputTestResultParser.IsPassedLine(line))
            {
                return(StandardOutputTestResultParser.CreatePassedTestResult(
                           testCase,
                           StandardOutputTestResultParser.ParseDuration(line, _logger)));
            }

            CrashedTestCase = testCase;
            string message = StandardOutputTestResultParser.CrashText;

            message += errorMsg == "" ? "" : $"\nTest output:\n\n{errorMsg}";
            TestResult result = StandardOutputTestResultParser.CreateFailedTestResult(
                testCase,
                TimeSpan.FromMilliseconds(0),
                message,
                "");

            return(result);
        }
        private List<TestResult> CollectTestResults(IEnumerable<TestCase> testCasesRun, string resultXmlFile, List<string> consoleOutput, string baseDir)
        {
            var testResults = new List<TestResult>();

            TestCase[] testCasesRunAsArray = testCasesRun as TestCase[] ?? testCasesRun.ToArray();
            var xmlParser = new XmlTestResultParser(testCasesRunAsArray, resultXmlFile, _testEnvironment, baseDir);
            var consoleParser = new StandardOutputTestResultParser(testCasesRunAsArray, consoleOutput, _testEnvironment, baseDir);

            testResults.AddRange(xmlParser.GetTestResults());
            _testEnvironment.DebugInfo($"Collected {testResults.Count} test results from XML result file '{resultXmlFile}'");

            if (testResults.Count < testCasesRunAsArray.Length)
            {
                List<TestResult> consoleResults = consoleParser.GetTestResults();
                int nrOfCollectedTestResults = 0;
                // ReSharper disable once AccessToModifiedClosure
                foreach (TestResult testResult in consoleResults.Where(tr => !testResults.Exists(tr2 => tr.TestCase.FullyQualifiedName == tr2.TestCase.FullyQualifiedName)))
                {
                    testResults.Add(testResult);
                    nrOfCollectedTestResults++;
                }
                _testEnvironment.DebugInfo($"Collected {nrOfCollectedTestResults} test results from console output");
            }

            if (testResults.Count < testCasesRunAsArray.Length)
            {
                string errorMessage, errorStackTrace = null;
                if (consoleParser.CrashedTestCase == null)
                {
                    errorMessage = "";
                }
                else
                {
                    errorMessage = $"reason is probably a crash of test {consoleParser.CrashedTestCase.DisplayName}";
                    errorStackTrace = ErrorMessageParser.CreateStackTraceEntry("crash suspect",
                        consoleParser.CrashedTestCase.CodeFilePath, consoleParser.CrashedTestCase.LineNumber.ToString());
                }
                int nrOfCreatedTestResults = 0;
                // ReSharper disable once AccessToModifiedClosure
                foreach (TestCase testCase in testCasesRunAsArray.Where(tc => !testResults.Exists(tr => tr.TestCase.FullyQualifiedName == tc.FullyQualifiedName)))
                {
                    testResults.Add(new TestResult(testCase)
                    {
                        ComputerName = Environment.MachineName,
                        Outcome = TestOutcome.Skipped,
                        ErrorMessage = errorMessage,
                        ErrorStackTrace = errorStackTrace
                    });
                    nrOfCreatedTestResults++;
                }
                _testEnvironment.DebugInfo($"Created {nrOfCreatedTestResults} test results for tests which were neither found in result XML file nor in console output");
            }

            testResults = testResults.OrderBy(tr => tr.TestCase.FullyQualifiedName).ToList();

            return testResults;
        }