private Enum FailedTestCaseMatched(Match match, TestResultParserStateContext stateContext)
        {
            var mochaStateContext = stateContext as MochaParserStateContext;

            // If a failed test case is encountered while in the summary state it indicates either completion
            // or corruption of summary. Since Summary is Gospel to us, we will ignore the latter and publish
            // the run regardless.
            this.attemptPublishAndResetParser();

            // Handling parse errors is unnecessary
            var testCaseNumber = int.Parse(match.Groups[RegexCaptureGroups.FailedTestCaseNumber].Value);

            // If it was not 1 there's a good chance we read some random line as a failed test case hence consider it a
            // as a match but do not add it to our list of test cases
            if (testCaseNumber != 1)
            {
                this.logger.Error($"MochaTestResultParser : ExpectingTestRunSummary : Expecting failed test case with" +
                                  $" number {mochaStateContext.LastFailedTestCaseNumber + 1} but found {testCaseNumber} instead");
                this.telemetryDataCollector.AddToCumulativeTelemetry(MochaTelemetryConstants.EventArea, MochaTelemetryConstants.UnexpectedFailedTestCaseNumber,
                                                                     new List <int> {
                    mochaStateContext.TestRun.TestRunId
                }, true);

                return(MochaParserStates.ExpectingTestResults);
            }

            // Increment either ways whether it was expected or context was reset and the encountered number was 1
            mochaStateContext.LastFailedTestCaseNumber++;

            var testResult = PrepareTestResult(TestOutcome.Failed, match);

            mochaStateContext.TestRun.FailedTests.Add(testResult);

            return(MochaParserStates.ExpectingTestResults);
        }
Esempio n. 2
0
        private Enum StackTraceStartMatched(Match match, TestResultParserStateContext stateContext)
        {
            var jestStateContext = stateContext as JestParserStateContext;

            if (jestStateContext.FailedTestsSummaryIndicatorEncountered)
            {
                logger.Verbose($"JestTestResultParser : ExpectingStackTraces: Ignoring StackTrace/Failed test case at line " +
                               $"{stateContext.CurrentLineNumber} as it is part of the summarized failures.");
                return(JestParserStates.ExpectingStackTraces);
            }

            // In non verbose mode console out appears as a failed test case
            // Only difference being it's not colored red
            if (match.Groups[RegexCaptureGroups.TestCaseName].Value == "Console")
            {
                logger.Verbose($"JestTestResultParser : ExpectingStackTraces: Ignoring apparent StackTrace/Failed test case at line " +
                               $"{stateContext.CurrentLineNumber} as Jest prints console out in this format in non verbose mode.");
                return(JestParserStates.ExpectingStackTraces);
            }

            var testResult = PrepareTestResult(TestOutcome.Failed, match);

            jestStateContext.TestRun.FailedTests.Add(testResult);

            return(JestParserStates.ExpectingStackTraces);
        }
Esempio n. 3
0
        private Enum PassedTestCaseMatched(Match match, TestResultParserStateContext stateContext)
        {
            var mochaStateContext = stateContext as MochaParserStateContext;

            // If a passed test case is encountered while in the stack traces state it indicates corruption
            // or incomplete stack trace data
            // This check is safety check for when we try to parse stack trace contents, as of now it will always evaluate to true
            if (mochaStateContext.StackTracesToSkipParsingPostSummary != 0)
            {
                this.logger.Error($"MochaTestResultParser : ExpectingStackTraces :  Expecting stack traces but found passed test case instead at line {mochaStateContext.CurrentLineNumber}.");
                this.telemetryDataCollector.AddToCumulativeTelemetry(MochaTelemetryConstants.EventArea, MochaTelemetryConstants.ExpectingStackTracesButFoundPassedTest,
                                                                     new List <int> {
                    mochaStateContext.TestRun.TestRunId
                }, true);
            }

            this.attemptPublishAndResetParser();

            var testResult = PrepareTestResult(TestOutcome.Passed, match);

            mochaStateContext.TestRun.PassedTests.Add(testResult);

            this.logger.Info($"MochaTestResultParser : ExpectingStackTraces : Transitioned to state ExpectingTestResults " +
                             $"at line {mochaStateContext.CurrentLineNumber}.");

            return(MochaParserStates.ExpectingTestResults);
        }
Esempio n. 4
0
        private Enum PassedTestsSummaryMatched(Match match, TestResultParserStateContext stateContext)
        {
            var mochaStateContext = stateContext as MochaParserStateContext;

            this.logger.Info($"MochaTestResultParser : ExpectingTestResults : Passed test summary encountered at line {mochaStateContext.CurrentLineNumber}.");

            mochaStateContext.LinesWithinWhichMatchIsExpected = 1;
            mochaStateContext.NextExpectedMatch        = "failed/pending tests summary";
            mochaStateContext.LastFailedTestCaseNumber = 0;

            // Handling parse errors is unnecessary
            var totalPassed = int.Parse(match.Groups[RegexCaptureGroups.PassedTests].Value);

            mochaStateContext.TestRun.TestRunSummary.TotalPassed = totalPassed;

            // Fire telemetry if summary does not agree with parsed tests count
            if (mochaStateContext.TestRun.TestRunSummary.TotalPassed != mochaStateContext.TestRun.PassedTests.Count)
            {
                this.logger.Error($"MochaTestResultParser : ExpectingTestResults : Passed tests count does not match passed summary" +
                                  $" at line {mochaStateContext.CurrentLineNumber}");
                this.telemetryDataCollector.AddToCumulativeTelemetry(MochaTelemetryConstants.EventArea,
                                                                     MochaTelemetryConstants.PassedSummaryMismatch, new List <int> {
                    mochaStateContext.TestRun.TestRunId
                }, true);
            }

            // Extract the test run time from the passed tests summary
            ExtractTestRunTime(match, mochaStateContext);

            this.logger.Info($"MochaTestResultParser : ExpectingTestResults : Transitioned to state ExpectingTestRunSummary" +
                             $" at line {mochaStateContext.CurrentLineNumber}.");
            return(MochaParserStates.ExpectingTestRunSummary);
        }
        private Enum TestRunTimeMatched(Match match, TestResultParserStateContext stateContext)
        {
            var jestStateContext = stateContext as JestParserStateContext;

            // Extract the test run time
            // Handling parse errors is unnecessary
            var timeTaken = double.Parse(match.Groups[RegexCaptureGroups.TestRunTime].Value);

            // Store time taken based on the unit used
            switch (match.Groups[RegexCaptureGroups.TestRunTimeUnit].Value)
            {
            case "ms":
                jestStateContext.TestRun.TestRunSummary.TotalExecutionTime = TimeSpan.FromMilliseconds(timeTaken);
                break;

            case "s":
                jestStateContext.TestRun.TestRunSummary.TotalExecutionTime = TimeSpan.FromMilliseconds(timeTaken * 1000);
                break;

            case "m":
                jestStateContext.TestRun.TestRunSummary.TotalExecutionTime = TimeSpan.FromMilliseconds(timeTaken * 60 * 1000);
                break;

            case "h":
                jestStateContext.TestRun.TestRunSummary.TotalExecutionTime = TimeSpan.FromMilliseconds(timeTaken * 60 * 60 * 1000);
                break;
            }

            this.attemptPublishAndResetParser();

            return(JestParserStates.ExpectingTestRunStart);
        }
Esempio n. 6
0
        private Enum TestRunStartMatched(Match match, TestResultParserStateContext stateContext)
        {
            var jestStateContext = stateContext as JestParserStateContext;

            // Do we want to use PASS/FAIL information here?

            return(JestParserStates.ExpectingTestResults);
        }
Esempio n. 7
0
        private Enum FailedTestsSummaryIndicatorMatched(Match match, TestResultParserStateContext stateContext)
        {
            var jestStateContext = stateContext as JestParserStateContext;

            jestStateContext.FailedTestsSummaryIndicatorEncountered = true;

            return(JestParserStates.ExpectingStackTraces);
        }
Esempio n. 8
0
        private Enum SummaryStartMatched(Match match, TestResultParserStateContext stateContext)
        {
            var jestStateContext = stateContext as JestParserStateContext;

            jestStateContext.LinesWithinWhichMatchIsExpected = 1;
            jestStateContext.NextExpectedMatch = "tests summary";

            return(JestParserStates.ExpectingTestRunSummary);
        }
Esempio n. 9
0
        private Enum PendingTestCaseMatched(Match match, TestResultParserStateContext stateContext)
        {
            var mochaStateContext = stateContext as MochaParserStateContext;

            var testResult = PrepareTestResult(TestOutcome.NotExecuted, match);

            mochaStateContext.TestRun.SkippedTests.Add(testResult);

            return(MochaParserStates.ExpectingTestResults);
        }
Esempio n. 10
0
        private Enum SummaryStartMatched(Match match, TestResultParserStateContext stateContext)
        {
            var jestStateContext = stateContext as JestParserStateContext;

            jestStateContext.LinesWithinWhichMatchIsExpected = 1;
            jestStateContext.NextExpectedMatch = "tests summary";

            logger.Info($"JestTestResultParser : ExpectingStackTraces : Transitioned to state ExpectingTestRunSummary.");

            return(JestParserStates.ExpectingTestRunSummary);
        }
Esempio n. 11
0
        private Enum FailedTestCaseMatched(Match match, TestResultParserStateContext stateContext)
        {
            var jestStateContext = stateContext as JestParserStateContext;

            // Used for telemetry for identifying how many runs are using --verbose option
            jestStateContext.VerboseOptionEnabled = true;

            // TODO: Revisit if we even need to match these, expcept for telemtry no other use
            // No-op as we would like to pick up failed test cases in the stack traces state

            return(JestParserStates.ExpectingTestResults);
        }
Esempio n. 12
0
        private Enum PassedTestCaseMatched(Match match, TestResultParserStateContext stateContext)
        {
            var jestStateContext = stateContext as JestParserStateContext;

            var testResult = PrepareTestResult(TestOutcome.Passed, match);

            jestStateContext.TestRun.PassedTests.Add(testResult);

            // Used for telemetry for identifying how many runs are using --verbose option
            jestStateContext.VerboseOptionEnabled = true;

            return(JestParserStates.ExpectingTestResults);
        }
        private Enum PendingTestCaseMatched(Match match, TestResultParserStateContext stateContext)
        {
            var mochaStateContext = stateContext as MochaParserStateContext;

            // If a pending test case is encountered while in the summary state it indicates either completion
            // or corruption of summary. Since Summary is Gospel to us, we will ignore the latter and publish
            // the run regardless.
            this.attemptPublishAndResetParser();

            var testResult = PrepareTestResult(TestOutcome.NotExecuted, match);

            mochaStateContext.TestRun.SkippedTests.Add(testResult);
            return(MochaParserStates.ExpectingTestResults);
        }
        private Enum PendingTestsSummaryMatched(Match match, TestResultParserStateContext stateContext)
        {
            var mochaStateContext = stateContext as MochaParserStateContext;

            this.logger.Info($"MochaTestResultParser : ExpectingTestRunSummary : Pending tests summary encountered at line {mochaStateContext.CurrentLineNumber}.");
            mochaStateContext.LinesWithinWhichMatchIsExpected = 1;
            mochaStateContext.NextExpectedMatch = "failed tests summary";

            // Handling parse errors is unnecessary
            var totalPending = int.Parse(match.Groups[RegexCaptureGroups.PendingTests].Value);

            mochaStateContext.TestRun.TestRunSummary.TotalSkipped = totalPending;

            return(MochaParserStates.ExpectingTestRunSummary);
        }
Esempio n. 15
0
        private Enum TestRunStartMatched(Match match, TestResultParserStateContext stateContext)
        {
            var jestStateContext = stateContext as JestParserStateContext;

            // If a test run start indicator is encountered after failedTestsSummaryInidicator has
            // been encountered it must be ignored
            if (jestStateContext.FailedTestsSummaryIndicatorEncountered)
            {
                return(JestParserStates.ExpectingStackTraces);
            }

            // Do we want to use PASS/FAIL information here?
            logger.Info($"JestTestResultParser : ExpectingStackTraces : Transitioned to state ExpectingTestResults.");

            return(JestParserStates.ExpectingTestResults);
        }
Esempio n. 16
0
        private Enum StackTraceStartMatched(Match match, TestResultParserStateContext stateContext)
        {
            var jestStateContext = stateContext as JestParserStateContext;

            // In non verbose mode console out appears as a failed test case
            // Only difference being it's not colored red
            if (match.Groups[RegexCaptureGroups.TestCaseName].Value == "Console")
            {
                return(JestParserStates.ExpectingStackTraces);
            }

            var testResult = PrepareTestResult(TestOutcome.Failed, match);

            jestStateContext.TestRun.FailedTests.Add(testResult);

            return(JestParserStates.ExpectingStackTraces);
        }
        private Enum PassedTestCaseMatched(Match match, TestResultParserStateContext stateContext)
        {
            var mochaStateContext = stateContext as MochaParserStateContext;

            // If a passed test case is encountered while in the summary state it indicates either completion
            // or corruption of summary. Since Summary is Gospel to us, we will ignore the latter and publish
            // the run regardless.
            this.attemptPublishAndResetParser();

            var testResult = PrepareTestResult(TestOutcome.Passed, match);

            mochaStateContext.TestRun.PassedTests.Add(testResult);
            this.logger.Info($"MochaTestResultParser : ExpectingTestRunSummary : Transitioned to state ExpectingTestResults " +
                             $"at line {mochaStateContext.CurrentLineNumber}.");

            return(MochaParserStates.ExpectingTestResults);
        }
        private Enum TestsSummaryMatched(Match match, TestResultParserStateContext stateContext)
        {
            var jestStateContext = stateContext as JestParserStateContext;

            jestStateContext.LinesWithinWhichMatchIsExpected = 2;
            jestStateContext.NextExpectedMatch = "test run time";

            // Handling parse errors is unnecessary
            int.TryParse(match.Groups[RegexCaptureGroups.PassedTests].Value, out int totalPassed);
            int.TryParse(match.Groups[RegexCaptureGroups.FailedTests].Value, out int totalFailed);
            int.TryParse(match.Groups[RegexCaptureGroups.SkippedTests].Value, out int totalSkipped);

            jestStateContext.TestRun.TestRunSummary.TotalPassed  = totalPassed;
            jestStateContext.TestRun.TestRunSummary.TotalFailed  = totalFailed;
            jestStateContext.TestRun.TestRunSummary.TotalSkipped = totalSkipped;

            return(JestParserStates.ExpectingTestRunSummary);
        }
        private Enum FailedTestsSummaryMatched(Match match, TestResultParserStateContext stateContext)
        {
            var mochaStateContext = stateContext as MochaParserStateContext;

            this.logger.Info($"MochaTestResultParser : ExpectingTestRunSummary : Failed tests summary encountered at line {mochaStateContext.CurrentLineNumber}.");
            mochaStateContext.LinesWithinWhichMatchIsExpected = 0;

            // Handling parse errors is unnecessary
            var totalFailed = int.Parse(match.Groups[RegexCaptureGroups.FailedTests].Value);

            mochaStateContext.TestRun.TestRunSummary.TotalFailed  = totalFailed;
            mochaStateContext.StackTracesToSkipParsingPostSummary = totalFailed;

            // Do we want transition logs here?
            this.logger.Info($"MochaTestResultParser : ExpectingTestRunSummary : Transitioned to state ExpectingStackTraces" +
                             $" at line {mochaStateContext.CurrentLineNumber}.");
            return(MochaParserStates.ExpectingStackTraces);
        }
Esempio n. 20
0
        private Enum FailedTestCaseMatched(Match match, TestResultParserStateContext stateContext)
        {
            var mochaStateContext = stateContext as MochaParserStateContext;

            // Handling parse errors is unnecessary
            var testCaseNumber = int.Parse(match.Groups[RegexCaptureGroups.FailedTestCaseNumber].Value);

            // In the event the failed test case number does not match the expected test case number log an error
            if (testCaseNumber != mochaStateContext.LastFailedTestCaseNumber + 1)
            {
                this.logger.Error($"MochaTestResultParser : ExpectingTestResults : Expecting failed test case with" +
                                  $" number {mochaStateContext.LastFailedTestCaseNumber + 1} but found {testCaseNumber} instead");
                this.telemetryDataCollector.AddToCumulativeTelemetry(MochaTelemetryConstants.EventArea, MochaTelemetryConstants.UnexpectedFailedTestCaseNumber,
                                                                     new List <int> {
                    mochaStateContext.TestRun.TestRunId
                }, true);

                // If it was not 1 there's a good chance we read some random line as a failed test case hence consider it a
                // as a match but do not add it to our list of test cases
                if (testCaseNumber != 1)
                {
                    return(MochaParserStates.ExpectingTestResults);
                }

                // If the number was 1 then there's a good chance this is the beginning of the next test run, hence reset and start over
                // This is something we might choose to change if we realize there is a chance we can get such false detections often in the middle of a run
                this.attemptPublishAndResetParser();
            }

            // Increment either ways whether it was expected or context was reset and the encountered number was 1
            mochaStateContext.LastFailedTestCaseNumber++;

            var testResult = PrepareTestResult(TestOutcome.Failed, match);

            mochaStateContext.TestRun.FailedTests.Add(testResult);

            return(MochaParserStates.ExpectingTestResults);
        }
Esempio n. 21
0
        private Enum FailedTestCaseMatched(Match match, TestResultParserStateContext stateContext)
        {
            var mochaStateContext = stateContext as MochaParserStateContext;

            // Handling parse errors is unnecessary
            var testCaseNumber = int.Parse(match.Groups[RegexCaptureGroups.FailedTestCaseNumber].Value);

            // In the event the failed test case number does not match the expected test case number log an error
            if (testCaseNumber != mochaStateContext.LastFailedTestCaseNumber + 1)
            {
                this.logger.Error($"MochaTestResultParser : ExpectingStackTraces : Expecting stack trace with" +
                                  $" number {mochaStateContext.LastFailedTestCaseNumber + 1} but found {testCaseNumber} instead");
                this.telemetryDataCollector.AddToCumulativeTelemetry(MochaTelemetryConstants.EventArea, MochaTelemetryConstants.UnexpectedFailedStackTraceNumber,
                                                                     new List <int> {
                    mochaStateContext.TestRun.TestRunId
                }, true);

                // If it was not 1 there's a good chance we read some random line as a failed test case hence consider it a
                // as a match but do not consider it a valid stack trace
                if (testCaseNumber != 1)
                {
                    // If we are parsing stack traces then we should not return this as
                    // a successful match. If we do so then stack trace addition will not
                    // happen for the current line
                    return(MochaParserStates.ExpectingStackTraces);
                }

                this.telemetryDataCollector.AddToCumulativeTelemetry(MochaTelemetryConstants.EventArea, MochaTelemetryConstants.AttemptPublishAndResetParser,
                                                                     new List <string> {
                    $"Expecting stack trace with number {mochaStateContext.LastFailedTestCaseNumber} but found {testCaseNumber} instead"
                });

                // If the number was 1 then there's a good chance this is the beginning of the next test run, hence reset and start over
                this.attemptPublishAndResetParser();

                mochaStateContext.LastFailedTestCaseNumber++;

                var testResult = PrepareTestResult(TestOutcome.Failed, match);
                mochaStateContext.TestRun.FailedTests.Add(testResult);

                this.logger.Info($"MochaTestResultParser : ExpectingStackTraces : Transitioned to state ExpectingTestResults " +
                                 $"at line {mochaStateContext.CurrentLineNumber}.");

                return(MochaParserStates.ExpectingTestResults);
            }

            mochaStateContext.LastFailedTestCaseNumber++;

            // As of now we are ignoring stack traces
            // Otherwise parsing stacktrace code will go here

            mochaStateContext.StackTracesToSkipParsingPostSummary--;

            if (mochaStateContext.StackTracesToSkipParsingPostSummary == 0)
            {
                // We can also choose to ignore extra failures post summary if the number is not 1
                this.attemptPublishAndResetParser();
                return(MochaParserStates.ExpectingTestResults);
            }

            return(MochaParserStates.ExpectingStackTraces);
        }