private Enum FailedTestCaseMatched(Match match, TestResultParserStateContext stateContext) { var mochaStateContext = stateContext as MochaParserStateContext; // If a failed test case is encountered while in the summary state it indicates either completion // or corruption of summary. Since Summary is Gospel to us, we will ignore the latter and publish // the run regardless. this.attemptPublishAndResetParser(); // Handling parse errors is unnecessary var testCaseNumber = int.Parse(match.Groups[RegexCaptureGroups.FailedTestCaseNumber].Value); // If it was not 1 there's a good chance we read some random line as a failed test case hence consider it a // as a match but do not add it to our list of test cases if (testCaseNumber != 1) { this.logger.Error($"MochaTestResultParser : ExpectingTestRunSummary : Expecting failed test case with" + $" number {mochaStateContext.LastFailedTestCaseNumber + 1} but found {testCaseNumber} instead"); this.telemetryDataCollector.AddToCumulativeTelemetry(MochaTelemetryConstants.EventArea, MochaTelemetryConstants.UnexpectedFailedTestCaseNumber, new List <int> { mochaStateContext.TestRun.TestRunId }, true); return(MochaParserStates.ExpectingTestResults); } // Increment either ways whether it was expected or context was reset and the encountered number was 1 mochaStateContext.LastFailedTestCaseNumber++; var testResult = PrepareTestResult(TestOutcome.Failed, match); mochaStateContext.TestRun.FailedTests.Add(testResult); return(MochaParserStates.ExpectingTestResults); }
private Enum StackTraceStartMatched(Match match, TestResultParserStateContext stateContext) { var jestStateContext = stateContext as JestParserStateContext; if (jestStateContext.FailedTestsSummaryIndicatorEncountered) { logger.Verbose($"JestTestResultParser : ExpectingStackTraces: Ignoring StackTrace/Failed test case at line " + $"{stateContext.CurrentLineNumber} as it is part of the summarized failures."); return(JestParserStates.ExpectingStackTraces); } // In non verbose mode console out appears as a failed test case // Only difference being it's not colored red if (match.Groups[RegexCaptureGroups.TestCaseName].Value == "Console") { logger.Verbose($"JestTestResultParser : ExpectingStackTraces: Ignoring apparent StackTrace/Failed test case at line " + $"{stateContext.CurrentLineNumber} as Jest prints console out in this format in non verbose mode."); return(JestParserStates.ExpectingStackTraces); } var testResult = PrepareTestResult(TestOutcome.Failed, match); jestStateContext.TestRun.FailedTests.Add(testResult); return(JestParserStates.ExpectingStackTraces); }
private Enum PassedTestCaseMatched(Match match, TestResultParserStateContext stateContext) { var mochaStateContext = stateContext as MochaParserStateContext; // If a passed test case is encountered while in the stack traces state it indicates corruption // or incomplete stack trace data // This check is safety check for when we try to parse stack trace contents, as of now it will always evaluate to true if (mochaStateContext.StackTracesToSkipParsingPostSummary != 0) { this.logger.Error($"MochaTestResultParser : ExpectingStackTraces : Expecting stack traces but found passed test case instead at line {mochaStateContext.CurrentLineNumber}."); this.telemetryDataCollector.AddToCumulativeTelemetry(MochaTelemetryConstants.EventArea, MochaTelemetryConstants.ExpectingStackTracesButFoundPassedTest, new List <int> { mochaStateContext.TestRun.TestRunId }, true); } this.attemptPublishAndResetParser(); var testResult = PrepareTestResult(TestOutcome.Passed, match); mochaStateContext.TestRun.PassedTests.Add(testResult); this.logger.Info($"MochaTestResultParser : ExpectingStackTraces : Transitioned to state ExpectingTestResults " + $"at line {mochaStateContext.CurrentLineNumber}."); return(MochaParserStates.ExpectingTestResults); }
private Enum PassedTestsSummaryMatched(Match match, TestResultParserStateContext stateContext) { var mochaStateContext = stateContext as MochaParserStateContext; this.logger.Info($"MochaTestResultParser : ExpectingTestResults : Passed test summary encountered at line {mochaStateContext.CurrentLineNumber}."); mochaStateContext.LinesWithinWhichMatchIsExpected = 1; mochaStateContext.NextExpectedMatch = "failed/pending tests summary"; mochaStateContext.LastFailedTestCaseNumber = 0; // Handling parse errors is unnecessary var totalPassed = int.Parse(match.Groups[RegexCaptureGroups.PassedTests].Value); mochaStateContext.TestRun.TestRunSummary.TotalPassed = totalPassed; // Fire telemetry if summary does not agree with parsed tests count if (mochaStateContext.TestRun.TestRunSummary.TotalPassed != mochaStateContext.TestRun.PassedTests.Count) { this.logger.Error($"MochaTestResultParser : ExpectingTestResults : Passed tests count does not match passed summary" + $" at line {mochaStateContext.CurrentLineNumber}"); this.telemetryDataCollector.AddToCumulativeTelemetry(MochaTelemetryConstants.EventArea, MochaTelemetryConstants.PassedSummaryMismatch, new List <int> { mochaStateContext.TestRun.TestRunId }, true); } // Extract the test run time from the passed tests summary ExtractTestRunTime(match, mochaStateContext); this.logger.Info($"MochaTestResultParser : ExpectingTestResults : Transitioned to state ExpectingTestRunSummary" + $" at line {mochaStateContext.CurrentLineNumber}."); return(MochaParserStates.ExpectingTestRunSummary); }
private Enum TestRunTimeMatched(Match match, TestResultParserStateContext stateContext) { var jestStateContext = stateContext as JestParserStateContext; // Extract the test run time // Handling parse errors is unnecessary var timeTaken = double.Parse(match.Groups[RegexCaptureGroups.TestRunTime].Value); // Store time taken based on the unit used switch (match.Groups[RegexCaptureGroups.TestRunTimeUnit].Value) { case "ms": jestStateContext.TestRun.TestRunSummary.TotalExecutionTime = TimeSpan.FromMilliseconds(timeTaken); break; case "s": jestStateContext.TestRun.TestRunSummary.TotalExecutionTime = TimeSpan.FromMilliseconds(timeTaken * 1000); break; case "m": jestStateContext.TestRun.TestRunSummary.TotalExecutionTime = TimeSpan.FromMilliseconds(timeTaken * 60 * 1000); break; case "h": jestStateContext.TestRun.TestRunSummary.TotalExecutionTime = TimeSpan.FromMilliseconds(timeTaken * 60 * 60 * 1000); break; } this.attemptPublishAndResetParser(); return(JestParserStates.ExpectingTestRunStart); }
private Enum TestRunStartMatched(Match match, TestResultParserStateContext stateContext) { var jestStateContext = stateContext as JestParserStateContext; // Do we want to use PASS/FAIL information here? return(JestParserStates.ExpectingTestResults); }
private Enum FailedTestsSummaryIndicatorMatched(Match match, TestResultParserStateContext stateContext) { var jestStateContext = stateContext as JestParserStateContext; jestStateContext.FailedTestsSummaryIndicatorEncountered = true; return(JestParserStates.ExpectingStackTraces); }
private Enum SummaryStartMatched(Match match, TestResultParserStateContext stateContext) { var jestStateContext = stateContext as JestParserStateContext; jestStateContext.LinesWithinWhichMatchIsExpected = 1; jestStateContext.NextExpectedMatch = "tests summary"; return(JestParserStates.ExpectingTestRunSummary); }
private Enum PendingTestCaseMatched(Match match, TestResultParserStateContext stateContext) { var mochaStateContext = stateContext as MochaParserStateContext; var testResult = PrepareTestResult(TestOutcome.NotExecuted, match); mochaStateContext.TestRun.SkippedTests.Add(testResult); return(MochaParserStates.ExpectingTestResults); }
private Enum SummaryStartMatched(Match match, TestResultParserStateContext stateContext) { var jestStateContext = stateContext as JestParserStateContext; jestStateContext.LinesWithinWhichMatchIsExpected = 1; jestStateContext.NextExpectedMatch = "tests summary"; logger.Info($"JestTestResultParser : ExpectingStackTraces : Transitioned to state ExpectingTestRunSummary."); return(JestParserStates.ExpectingTestRunSummary); }
private Enum FailedTestCaseMatched(Match match, TestResultParserStateContext stateContext) { var jestStateContext = stateContext as JestParserStateContext; // Used for telemetry for identifying how many runs are using --verbose option jestStateContext.VerboseOptionEnabled = true; // TODO: Revisit if we even need to match these, expcept for telemtry no other use // No-op as we would like to pick up failed test cases in the stack traces state return(JestParserStates.ExpectingTestResults); }
private Enum PassedTestCaseMatched(Match match, TestResultParserStateContext stateContext) { var jestStateContext = stateContext as JestParserStateContext; var testResult = PrepareTestResult(TestOutcome.Passed, match); jestStateContext.TestRun.PassedTests.Add(testResult); // Used for telemetry for identifying how many runs are using --verbose option jestStateContext.VerboseOptionEnabled = true; return(JestParserStates.ExpectingTestResults); }
private Enum PendingTestCaseMatched(Match match, TestResultParserStateContext stateContext) { var mochaStateContext = stateContext as MochaParserStateContext; // If a pending test case is encountered while in the summary state it indicates either completion // or corruption of summary. Since Summary is Gospel to us, we will ignore the latter and publish // the run regardless. this.attemptPublishAndResetParser(); var testResult = PrepareTestResult(TestOutcome.NotExecuted, match); mochaStateContext.TestRun.SkippedTests.Add(testResult); return(MochaParserStates.ExpectingTestResults); }
private Enum PendingTestsSummaryMatched(Match match, TestResultParserStateContext stateContext) { var mochaStateContext = stateContext as MochaParserStateContext; this.logger.Info($"MochaTestResultParser : ExpectingTestRunSummary : Pending tests summary encountered at line {mochaStateContext.CurrentLineNumber}."); mochaStateContext.LinesWithinWhichMatchIsExpected = 1; mochaStateContext.NextExpectedMatch = "failed tests summary"; // Handling parse errors is unnecessary var totalPending = int.Parse(match.Groups[RegexCaptureGroups.PendingTests].Value); mochaStateContext.TestRun.TestRunSummary.TotalSkipped = totalPending; return(MochaParserStates.ExpectingTestRunSummary); }
private Enum TestRunStartMatched(Match match, TestResultParserStateContext stateContext) { var jestStateContext = stateContext as JestParserStateContext; // If a test run start indicator is encountered after failedTestsSummaryInidicator has // been encountered it must be ignored if (jestStateContext.FailedTestsSummaryIndicatorEncountered) { return(JestParserStates.ExpectingStackTraces); } // Do we want to use PASS/FAIL information here? logger.Info($"JestTestResultParser : ExpectingStackTraces : Transitioned to state ExpectingTestResults."); return(JestParserStates.ExpectingTestResults); }
private Enum StackTraceStartMatched(Match match, TestResultParserStateContext stateContext) { var jestStateContext = stateContext as JestParserStateContext; // In non verbose mode console out appears as a failed test case // Only difference being it's not colored red if (match.Groups[RegexCaptureGroups.TestCaseName].Value == "Console") { return(JestParserStates.ExpectingStackTraces); } var testResult = PrepareTestResult(TestOutcome.Failed, match); jestStateContext.TestRun.FailedTests.Add(testResult); return(JestParserStates.ExpectingStackTraces); }
private Enum PassedTestCaseMatched(Match match, TestResultParserStateContext stateContext) { var mochaStateContext = stateContext as MochaParserStateContext; // If a passed test case is encountered while in the summary state it indicates either completion // or corruption of summary. Since Summary is Gospel to us, we will ignore the latter and publish // the run regardless. this.attemptPublishAndResetParser(); var testResult = PrepareTestResult(TestOutcome.Passed, match); mochaStateContext.TestRun.PassedTests.Add(testResult); this.logger.Info($"MochaTestResultParser : ExpectingTestRunSummary : Transitioned to state ExpectingTestResults " + $"at line {mochaStateContext.CurrentLineNumber}."); return(MochaParserStates.ExpectingTestResults); }
private Enum TestsSummaryMatched(Match match, TestResultParserStateContext stateContext) { var jestStateContext = stateContext as JestParserStateContext; jestStateContext.LinesWithinWhichMatchIsExpected = 2; jestStateContext.NextExpectedMatch = "test run time"; // Handling parse errors is unnecessary int.TryParse(match.Groups[RegexCaptureGroups.PassedTests].Value, out int totalPassed); int.TryParse(match.Groups[RegexCaptureGroups.FailedTests].Value, out int totalFailed); int.TryParse(match.Groups[RegexCaptureGroups.SkippedTests].Value, out int totalSkipped); jestStateContext.TestRun.TestRunSummary.TotalPassed = totalPassed; jestStateContext.TestRun.TestRunSummary.TotalFailed = totalFailed; jestStateContext.TestRun.TestRunSummary.TotalSkipped = totalSkipped; return(JestParserStates.ExpectingTestRunSummary); }
private Enum FailedTestsSummaryMatched(Match match, TestResultParserStateContext stateContext) { var mochaStateContext = stateContext as MochaParserStateContext; this.logger.Info($"MochaTestResultParser : ExpectingTestRunSummary : Failed tests summary encountered at line {mochaStateContext.CurrentLineNumber}."); mochaStateContext.LinesWithinWhichMatchIsExpected = 0; // Handling parse errors is unnecessary var totalFailed = int.Parse(match.Groups[RegexCaptureGroups.FailedTests].Value); mochaStateContext.TestRun.TestRunSummary.TotalFailed = totalFailed; mochaStateContext.StackTracesToSkipParsingPostSummary = totalFailed; // Do we want transition logs here? this.logger.Info($"MochaTestResultParser : ExpectingTestRunSummary : Transitioned to state ExpectingStackTraces" + $" at line {mochaStateContext.CurrentLineNumber}."); return(MochaParserStates.ExpectingStackTraces); }
private Enum FailedTestCaseMatched(Match match, TestResultParserStateContext stateContext) { var mochaStateContext = stateContext as MochaParserStateContext; // Handling parse errors is unnecessary var testCaseNumber = int.Parse(match.Groups[RegexCaptureGroups.FailedTestCaseNumber].Value); // In the event the failed test case number does not match the expected test case number log an error if (testCaseNumber != mochaStateContext.LastFailedTestCaseNumber + 1) { this.logger.Error($"MochaTestResultParser : ExpectingTestResults : Expecting failed test case with" + $" number {mochaStateContext.LastFailedTestCaseNumber + 1} but found {testCaseNumber} instead"); this.telemetryDataCollector.AddToCumulativeTelemetry(MochaTelemetryConstants.EventArea, MochaTelemetryConstants.UnexpectedFailedTestCaseNumber, new List <int> { mochaStateContext.TestRun.TestRunId }, true); // If it was not 1 there's a good chance we read some random line as a failed test case hence consider it a // as a match but do not add it to our list of test cases if (testCaseNumber != 1) { return(MochaParserStates.ExpectingTestResults); } // If the number was 1 then there's a good chance this is the beginning of the next test run, hence reset and start over // This is something we might choose to change if we realize there is a chance we can get such false detections often in the middle of a run this.attemptPublishAndResetParser(); } // Increment either ways whether it was expected or context was reset and the encountered number was 1 mochaStateContext.LastFailedTestCaseNumber++; var testResult = PrepareTestResult(TestOutcome.Failed, match); mochaStateContext.TestRun.FailedTests.Add(testResult); return(MochaParserStates.ExpectingTestResults); }
private Enum FailedTestCaseMatched(Match match, TestResultParserStateContext stateContext) { var mochaStateContext = stateContext as MochaParserStateContext; // Handling parse errors is unnecessary var testCaseNumber = int.Parse(match.Groups[RegexCaptureGroups.FailedTestCaseNumber].Value); // In the event the failed test case number does not match the expected test case number log an error if (testCaseNumber != mochaStateContext.LastFailedTestCaseNumber + 1) { this.logger.Error($"MochaTestResultParser : ExpectingStackTraces : Expecting stack trace with" + $" number {mochaStateContext.LastFailedTestCaseNumber + 1} but found {testCaseNumber} instead"); this.telemetryDataCollector.AddToCumulativeTelemetry(MochaTelemetryConstants.EventArea, MochaTelemetryConstants.UnexpectedFailedStackTraceNumber, new List <int> { mochaStateContext.TestRun.TestRunId }, true); // If it was not 1 there's a good chance we read some random line as a failed test case hence consider it a // as a match but do not consider it a valid stack trace if (testCaseNumber != 1) { // If we are parsing stack traces then we should not return this as // a successful match. If we do so then stack trace addition will not // happen for the current line return(MochaParserStates.ExpectingStackTraces); } this.telemetryDataCollector.AddToCumulativeTelemetry(MochaTelemetryConstants.EventArea, MochaTelemetryConstants.AttemptPublishAndResetParser, new List <string> { $"Expecting stack trace with number {mochaStateContext.LastFailedTestCaseNumber} but found {testCaseNumber} instead" }); // If the number was 1 then there's a good chance this is the beginning of the next test run, hence reset and start over this.attemptPublishAndResetParser(); mochaStateContext.LastFailedTestCaseNumber++; var testResult = PrepareTestResult(TestOutcome.Failed, match); mochaStateContext.TestRun.FailedTests.Add(testResult); this.logger.Info($"MochaTestResultParser : ExpectingStackTraces : Transitioned to state ExpectingTestResults " + $"at line {mochaStateContext.CurrentLineNumber}."); return(MochaParserStates.ExpectingTestResults); } mochaStateContext.LastFailedTestCaseNumber++; // As of now we are ignoring stack traces // Otherwise parsing stacktrace code will go here mochaStateContext.StackTracesToSkipParsingPostSummary--; if (mochaStateContext.StackTracesToSkipParsingPostSummary == 0) { // We can also choose to ignore extra failures post summary if the number is not 1 this.attemptPublishAndResetParser(); return(MochaParserStates.ExpectingTestResults); } return(MochaParserStates.ExpectingStackTraces); }