private Enum PassedTestCaseMatched(Match match, AbstractParserStateContext stateContext) { var mochaStateContext = stateContext as MochaParserStateContext; // If a passed test case is encountered while in the stack traces state it indicates corruption // or incomplete stack trace data // This check is safety check for when we try to parse stack trace contents, as of now it will always evaluate to true if (mochaStateContext.StackTracesToExpectPostSummary != 0) { Logger.Error($"{ParserName} : {StateName} : Expecting stack traces but found passed test case instead at line {mochaStateContext.CurrentLineNumber}."); Telemetry.AddAndAggregate(MochaTelemetryConstants.ExpectingStackTracesButFoundPassedTest, new List <int> { mochaStateContext.TestRun.TestRunId }, MochaTelemetryConstants.EventArea); } AttemptPublishAndResetParser(); var testResult = PrepareTestResult(TestOutcome.Passed, match); mochaStateContext.TestRun.PassedTests.Add(testResult); Logger.Info($"{ParserName} : {StateName} : Transitioned to state ExpectingTestResults " + $"at line {mochaStateContext.CurrentLineNumber}."); return(MochaParserStates.ExpectingTestResults); }
private Enum SummaryMatched(Match match, AbstractParserStateContext stateContext) { var jasmineStateContext = stateContext as JasmineParserStateContext; jasmineStateContext.LinesWithinWhichMatchIsExpected = 1; jasmineStateContext.NextExpectedMatch = "test run time"; Logger.Info($"{ParserName} : {StateName} : Transitioned to state ExpectingTestRunSummary" + $" at line {jasmineStateContext.CurrentLineNumber}."); int.TryParse(match.Groups[RegexCaptureGroups.TotalTests].Value, out int totalTests); int.TryParse(match.Groups[RegexCaptureGroups.FailedTests].Value, out int failedTests); int.TryParse(match.Groups[RegexCaptureGroups.SkippedTests].Value, out int skippedTests); // Since suite errors are added as failures in the summary, we need to remove this from passedTests // calculation. var passedTests = totalTests - skippedTests - (failedTests - jasmineStateContext.SuiteErrors); jasmineStateContext.TestRun.TestRunSummary.TotalTests = totalTests; jasmineStateContext.TestRun.TestRunSummary.TotalFailed = failedTests; jasmineStateContext.TestRun.TestRunSummary.TotalSkipped = skippedTests; jasmineStateContext.TestRun.TestRunSummary.TotalPassed = passedTests; return(JasmineParserStates.ExpectingTestRunSummary); }
private Enum FailedTestCaseMatched(Match match, AbstractParserStateContext stateContext) { var mochaStateContext = stateContext as MochaParserStateContext; // If a failed test case is encountered while in the summary state it indicates either completion // or corruption of summary. Since Summary is Gospel to us, we will ignore the latter and publish // the run regardless. AttemptPublishAndResetParser(); // Handling parse errors is unnecessary var testCaseNumber = int.Parse(match.Groups[RegexCaptureGroups.FailedTestCaseNumber].Value); // If it was not 1 there's a good chance we read some random line as a failed test case hence consider it a // as a match but do not add it to our list of test cases if (testCaseNumber != 1) { Logger.Error($"{ParserName} : {StateName} : Expecting failed test case with" + $" number {mochaStateContext.LastFailedTestCaseNumber + 1} but found {testCaseNumber} instead"); Telemetry.AddAndAggregate(MochaTelemetryConstants.UnexpectedFailedTestCaseNumber, new List <int> { mochaStateContext.TestRun.TestRunId }, MochaTelemetryConstants.EventArea); return(MochaParserStates.ExpectingTestResults); } // Increment either ways whether it was expected or context was reset and the encountered number was 1 mochaStateContext.LastFailedTestCaseNumber++; var testResult = PrepareTestResult(TestOutcome.Failed, match); mochaStateContext.TestRun.FailedTests.Add(testResult); return(MochaParserStates.ExpectingTestResults); }
private Enum TestRunTimeMatched(Match match, AbstractParserStateContext stateContext) { var jestStateContext = stateContext as JestParserStateContext; // Extract the test run time // Handling parse errors is unnecessary var timeTaken = double.Parse(match.Groups[RegexCaptureGroups.TestRunTime].Value); // Store time taken based on the unit used switch (match.Groups[RegexCaptureGroups.TestRunTimeUnit].Value) { case "ms": jestStateContext.TestRun.TestRunSummary.TotalExecutionTime = TimeSpan.FromMilliseconds(timeTaken); break; case "s": jestStateContext.TestRun.TestRunSummary.TotalExecutionTime = TimeSpan.FromMilliseconds(timeTaken * 1000); break; case "m": jestStateContext.TestRun.TestRunSummary.TotalExecutionTime = TimeSpan.FromMilliseconds(timeTaken * 60 * 1000); break; case "h": jestStateContext.TestRun.TestRunSummary.TotalExecutionTime = TimeSpan.FromMilliseconds(timeTaken * 60 * 60 * 1000); break; } AttemptPublishAndResetParser(); return(JestParserStates.ExpectingTestRunStart); }
private Enum StackTraceStartMatched(Match match, AbstractParserStateContext stateContext) { var jestStateContext = stateContext as JestParserStateContext; // Set this by default to -1, if a genuine stack trace was encountered then the actual index will be set. jestStateContext.CurrentStackTraceIndex = -1; // In non verbose mode console out appears as a failed test case // Only difference being it's not colored red if (match.Groups[RegexCaptureGroups.TestCaseName].Value == "Console") { return(JestParserStates.ExpectingStackTraces); } var testResult = PrepareTestResult(TestOutcome.Failed, match); jestStateContext.TestRun.FailedTests.Add(testResult); jestStateContext.CurrentStackTraceIndex = jestStateContext.TestRun.FailedTests.Count - 1; // Expect the stack trace to not be more than 50 lines long // This is to ensure we don't skip publishing the run if the stack traces appear corrupted jestStateContext.LinesWithinWhichMatchIsExpected = 50; jestStateContext.NextExpectedMatch = "next stacktraceStart/testrunStart/testrunSummary"; jestStateContext.TestRun.FailedTests[jestStateContext.CurrentStackTraceIndex].StackTrace = match.Value; Logger.Info($"{ParserName} : {StateName} : Transitioned to state ExpectingStackTraces" + $" at line {jestStateContext.CurrentLineNumber}."); return(JestParserStates.ExpectingStackTraces); }
private Enum PassedTestsSummaryMatched(Match match, AbstractParserStateContext stateContext) { var mochaStateContext = stateContext as MochaParserStateContext; Logger.Info($"{ParserName} : {StateName} : Passed test summary encountered at line {mochaStateContext.CurrentLineNumber}."); mochaStateContext.LinesWithinWhichMatchIsExpected = 1; mochaStateContext.NextExpectedMatch = "failed/pending tests summary"; mochaStateContext.LastFailedTestCaseNumber = 0; // Handling parse errors is unnecessary var totalPassed = int.Parse(match.Groups[RegexCaptureGroups.PassedTests].Value); mochaStateContext.TestRun.TestRunSummary.TotalPassed = totalPassed; // Fire telemetry if summary does not agree with parsed tests count if (mochaStateContext.TestRun.TestRunSummary.TotalPassed != mochaStateContext.TestRun.PassedTests.Count) { Logger.Error($"{ParserName} : {StateName} : Passed tests count does not match passed summary" + $" at line {mochaStateContext.CurrentLineNumber}"); Telemetry.AddAndAggregate(MochaTelemetryConstants.PassedSummaryMismatch, new List <int> { mochaStateContext.TestRun.TestRunId }, MochaTelemetryConstants.EventArea); } // Extract the test run time from the passed tests summary ExtractTestRunTime(match, mochaStateContext); Logger.Info($"{ParserName} : {StateName} : Transitioned to state ExpectingTestRunSummary" + $" at line {mochaStateContext.CurrentLineNumber}."); return(MochaParserStates.ExpectingTestRunSummary); }
private Enum TestRunStartMatched(Match match, AbstractParserStateContext stateContext) { var jestStateContext = stateContext as JestParserStateContext; Logger.Info($"{ParserName} : {StateName} : Transitioned to state ExpectingTestResults" + $" at line {jestStateContext.CurrentLineNumber}."); return(JestParserStates.ExpectingTestResults); }
private Enum FailuresStartMatched(Match match, AbstractParserStateContext stateContext) { // All failures are reported after FailureStart regex is matched. var jasmineStateContext = stateContext as JasmineParserStateContext; jasmineStateContext.FailureStarterMatched = true; jasmineStateContext.PendingStarterMatched = false; return(JasmineParserStates.ExpectingTestResults); }
private Enum FailedTestsSummaryIndicatorMatched(Match match, AbstractParserStateContext stateContext) { var jestStateContext = stateContext as JestParserStateContext; jestStateContext.FailedTestsSummaryIndicatorEncountered = true; Logger.Info($"{ParserName} : {StateName} : Matched the failed tests summary indicator" + $" at line {jestStateContext.CurrentLineNumber}."); return(JestParserStates.ExpectingStackTraces); }
private Enum PendingTestCaseMatched(Match match, AbstractParserStateContext stateContext) { var mochaStateContext = stateContext as MochaParserStateContext; var testResult = PrepareTestResult(TestOutcome.NotExecuted, match); mochaStateContext.TestRun.SkippedTests.Add(testResult); return(MochaParserStates.ExpectingTestResults); }
private Enum FailedTestCaseMatched(Match match, AbstractParserStateContext stateContext) { var jestStateContext = stateContext as JestParserStateContext; // Used for telemetry for identifying how many runs are using --verbose option jestStateContext.VerboseOptionEnabled = true; // TODO: Revisit if we even need to match these, expcept for telemtry no other use // No-op as we would like to pick up failed test cases in the stack traces state return(JestParserStates.ExpectingTestResults); }
private Enum PendingStartMatched(Match match, AbstractParserStateContext stateContext) { // All pending are reported after PendingStart regex is matched. var jasmineStateContext = stateContext as JasmineParserStateContext; // We set this as true so that any failedOrpending regex match after pending starter matched will be reported as pending tests // as pending and failed have the same regex jasmineStateContext.PendingStarterMatched = true; jasmineStateContext.FailureStarterMatched = false; return(JasmineParserStates.ExpectingTestResults); }
private Enum SummaryStartMatched(Match match, AbstractParserStateContext stateContext) { var jestStateContext = stateContext as JestParserStateContext; jestStateContext.LinesWithinWhichMatchIsExpected = 1; jestStateContext.NextExpectedMatch = "tests summary"; Logger.Info($"{ParserName} : {StateName} : Transitioned to state ExpectingTestRunSummary" + $" at line {jestStateContext.CurrentLineNumber}."); return(JestParserStates.ExpectingTestRunSummary); }
private Enum TestRunStartMatched(Match match, AbstractParserStateContext stateContext) { var jasmineStateContext = stateContext as JasmineParserStateContext; Logger.Info($"{ParserName} : {StateName} : Resetting the parser, test run start matched unexpectedly, transitioned to state ExpectingTestResults" + $" at line {jasmineStateContext.CurrentLineNumber}."); // Test Run Start matched after already encountering test run start. // Parser should be reset. AttemptPublishAndResetParser(); return(JasmineParserStates.ExpectingTestResults); }
private Enum PassedTestCaseMatched(Match match, AbstractParserStateContext stateContext) { var jestStateContext = stateContext as JestParserStateContext; var testResult = PrepareTestResult(TestOutcome.Passed, match); jestStateContext.TestRun.PassedTests.Add(testResult); // Used for telemetry for identifying how many runs are using --verbose option jestStateContext.VerboseOptionEnabled = true; return(JestParserStates.ExpectingTestResults); }
private Enum TestRunStartMatched(Match match, AbstractParserStateContext stateContext) { var jasmineStateContext = stateContext as JasmineParserStateContext; Logger.Info($"{ParserName} : {StateName} : Transitioned to state ExpectingTestResults" + $" at line {jasmineStateContext.CurrentLineNumber}."); // Console logs are dumped after test run start, if the summary or failed/pending test do not appear withing // 500 line we wish to reset the parser to not incur unnecessary costs jasmineStateContext.LinesWithinWhichMatchIsExpected = 500; jasmineStateContext.NextExpectedMatch = "failed/pending tests or test run summary"; return(JasmineParserStates.ExpectingTestResults); }
private Enum TestRunStartMatched(Match match, AbstractParserStateContext stateContext) { var jasmineStateContext = stateContext as JasmineParserStateContext; Logger.Info($"{ParserName} : {StateName} : Resetting the parser, test run start matched unexpectedly, transitioned to state ExpectingTestResults" + $" at line {jasmineStateContext.CurrentLineNumber}."); // If a test run started is encountered while in the summary state it indicates either completion // or corruption of summary. Since Summary is Gospel to us, we will ignore the latter and publish // the run regardless. AttemptPublishAndResetParser(); return(JasmineParserStates.ExpectingTestResults); }
private Enum PendingTestCaseMatched(Match match, AbstractParserStateContext stateContext) { var mochaStateContext = stateContext as MochaParserStateContext; // If a pending test case is encountered while in the summary state it indicates either completion // or corruption of summary. Since Summary is Gospel to us, we will ignore the latter and publish // the run regardless. AttemptPublishAndResetParser(); var testResult = PrepareTestResult(TestOutcome.NotExecuted, match); mochaStateContext.TestRun.SkippedTests.Add(testResult); return(MochaParserStates.ExpectingTestResults); }
private Enum PendingTestsSummaryMatched(Match match, AbstractParserStateContext stateContext) { var mochaStateContext = stateContext as MochaParserStateContext; Logger.Info($"{ParserName} : {StateName} : Pending tests summary encountered at line {mochaStateContext.CurrentLineNumber}."); mochaStateContext.LinesWithinWhichMatchIsExpected = 1; mochaStateContext.NextExpectedMatch = "failed tests summary"; // Handling parse errors is unnecessary var totalPending = int.Parse(match.Groups[RegexCaptureGroups.PendingTests].Value); mochaStateContext.TestRun.TestRunSummary.TotalSkipped = totalPending; return(MochaParserStates.ExpectingTestRunSummary); }
private Enum TestRunTimeMatched(Match match, AbstractParserStateContext stateContext) { var jasmineStateContext = stateContext as JasmineParserStateContext; var timeTaken = double.Parse(match.Groups[RegexCaptureGroups.TestRunTime].Value); jasmineStateContext.TestRun.TestRunSummary.TotalExecutionTime = TimeSpan.FromMilliseconds(timeTaken * 1000); jasmineStateContext.IsTimeParsed = true; Logger.Info($"{ParserName} : {StateName} : Test run time matched, transitioned to state ExpectingTestRunStart" + $" at line {jasmineStateContext.CurrentLineNumber}."); AttemptPublishAndResetParser(); return(JasmineParserStates.ExpectingTestRunStart); }
private Enum TestRunStartMatched(Match match, AbstractParserStateContext stateContext) { var jestStateContext = stateContext as JestParserStateContext; Logger.Error($"{ParserName} : {StateName} : Transitioned to state ExpectingTestResults" + $" at line {jestStateContext.CurrentLineNumber} as test run start indicator was encountered before encountering" + $" the full summary."); Telemetry.AddAndAggregate(JestTelemetryConstants.UnexpectedTestRunStart, new List <int> { jestStateContext.TestRun.TestRunId }, JestTelemetryConstants.EventArea); AttemptPublishAndResetParser(); return(JestParserStates.ExpectingTestResults); }
private Enum TestRunStartMatched(Match match, AbstractParserStateContext stateContext) { var jestStateContext = stateContext as JestParserStateContext; // If a test run start indicator is encountered after failedTestsSummaryInidicator has // been encountered it must be ignored if (jestStateContext.FailedTestsSummaryIndicatorEncountered) { return(JestParserStates.ExpectingStackTraces); } Logger.Info($"{ParserName} : {StateName} : Transitioned to state ExpectingTestResults" + $" at line {jestStateContext.CurrentLineNumber}."); return(JestParserStates.ExpectingTestResults); }
/// <summary> /// If none of the patterns matched then considers adding the current line to stack trace /// based on whether a stack trace start has been encountered /// </summary> /// <param name="line">Current line</param> /// <param name="stateContext">State context object containing information of the parser's state</param> /// <returns>True if the parser was reset</returns> public override bool PeformNoPatternMatchedAction(string line, AbstractParserStateContext stateContext) { if (base.PeformNoPatternMatchedAction(line, stateContext)) { return(true); } var mochaStateContext = stateContext as MochaParserStateContext; if (mochaStateContext.CurrentStackTraceIndex > -1 && mochaStateContext.CurrentStackTraceIndex < stateContext.TestRun.FailedTests.Count) { stateContext.TestRun.FailedTests[mochaStateContext.CurrentStackTraceIndex].StackTrace += Environment.NewLine + line; } return(false); }
private Enum PassedTestCaseMatched(Match match, AbstractParserStateContext stateContext) { var mochaStateContext = stateContext as MochaParserStateContext; // If a passed test case is encountered while in the summary state it indicates either completion // or corruption of summary. Since Summary is Gospel to us, we will ignore the latter and publish // the run regardless. AttemptPublishAndResetParser(); var testResult = PrepareTestResult(TestOutcome.Passed, match); mochaStateContext.TestRun.PassedTests.Add(testResult); Logger.Info($"{ParserName} : {StateName} : Transitioned to state ExpectingTestResults " + $"at line {mochaStateContext.CurrentLineNumber}."); return(MochaParserStates.ExpectingTestResults); }
/// <summary> /// If none of the patterns matched then considers adding the current line to stack trace /// based on whether a stack trace start has been encountered /// </summary> /// <param name="line">Current line</param> /// <param name="stateContext">State context object containing information of the parser's state</param> /// <returns>True if the parser was reset</returns> public override bool PeformNoPatternMatchedAction(string line, AbstractParserStateContext stateContext) { if (base.PeformNoPatternMatchedAction(line, stateContext)) { return(true); } var jasmineStateContext = stateContext as JasmineParserStateContext; // Index out of range can never occur as the stack traces immediately follow the failed test case if (jasmineStateContext.CurrentStackTraceIndex > -1) { stateContext.TestRun.FailedTests[jasmineStateContext.CurrentStackTraceIndex].StackTrace += Environment.NewLine + line; } return(false); }
private Enum TestsSummaryMatched(Match match, AbstractParserStateContext stateContext) { var jestStateContext = stateContext as JestParserStateContext; jestStateContext.LinesWithinWhichMatchIsExpected = 2; jestStateContext.NextExpectedMatch = "test run time"; // Handling parse errors is unnecessary int.TryParse(match.Groups[RegexCaptureGroups.PassedTests].Value, out int totalPassed); int.TryParse(match.Groups[RegexCaptureGroups.FailedTests].Value, out int totalFailed); int.TryParse(match.Groups[RegexCaptureGroups.SkippedTests].Value, out int totalSkipped); int.TryParse(match.Groups[RegexCaptureGroups.TotalTests].Value, out int totalTests); jestStateContext.TestRun.TestRunSummary.TotalPassed = totalPassed; jestStateContext.TestRun.TestRunSummary.TotalFailed = totalFailed; jestStateContext.TestRun.TestRunSummary.TotalSkipped = totalSkipped; jestStateContext.TestRun.TestRunSummary.TotalTests = totalTests; return(JestParserStates.ExpectingTestRunSummary); }
private Enum FailedTestsSummaryMatched(Match match, AbstractParserStateContext stateContext) { var mochaStateContext = stateContext as MochaParserStateContext; Logger.Info($"{ParserName} : {StateName} : Failed tests summary encountered at line {mochaStateContext.CurrentLineNumber}."); // Handling parse errors is unnecessary var totalFailed = int.Parse(match.Groups[RegexCaptureGroups.FailedTests].Value); mochaStateContext.TestRun.TestRunSummary.TotalFailed = totalFailed; mochaStateContext.StackTracesToExpectPostSummary = totalFailed; // Max expected gap between summary and first stack trace mochaStateContext.LinesWithinWhichMatchIsExpected = 50; // Do we want transition logs here? Logger.Info($"{ParserName} : {StateName} : Transitioned to state ExpectingStackTraces" + $" at line {mochaStateContext.CurrentLineNumber}."); return(MochaParserStates.ExpectingStackTraces); }
private Enum SuiteErrorMatched(Match match, AbstractParserStateContext stateContext) { var jasmineStateContext = stateContext as JasmineParserStateContext; // Suite error is counted as failed and summary includes this while reporting var testResult = PrepareTestResult(TestOutcome.Failed, match); jasmineStateContext.TestRun.FailedTests.Add(testResult); jasmineStateContext.SuiteErrors++; jasmineStateContext.CurrentStackTraceIndex = jasmineStateContext.TestRun.FailedTests.Count - 1; // Expect the stack trace to not be more than 50 lines long // This is to ensure we don't skip publishing the run if the stack traces appear corrupted jasmineStateContext.LinesWithinWhichMatchIsExpected = 50; jasmineStateContext.NextExpectedMatch = "next failed test case or pending test cases start or test run summary"; jasmineStateContext.TestRun.FailedTests[jasmineStateContext.CurrentStackTraceIndex].StackTrace = match.Value; return(JasmineParserStates.ExpectingTestResults); }
private Enum FailedTestCaseMatched(Match match, AbstractParserStateContext stateContext) { var mochaStateContext = stateContext as MochaParserStateContext; // Handling parse errors is unnecessary var testCaseNumber = int.Parse(match.Groups[RegexCaptureGroups.FailedTestCaseNumber].Value); // In the event the failed test case number does not match the expected test case number log an error if (testCaseNumber != mochaStateContext.LastFailedTestCaseNumber + 1) { Logger.Error($"{ParserName} : {StateName} : Expecting failed test case with" + $" number {mochaStateContext.LastFailedTestCaseNumber + 1} but found {testCaseNumber} instead"); Telemetry.AddAndAggregate(MochaTelemetryConstants.UnexpectedFailedTestCaseNumber, new List <int> { mochaStateContext.TestRun.TestRunId }, MochaTelemetryConstants.EventArea); // If it was not 1 there's a good chance we read some random line as a failed test case hence consider it a // as a match but do not add it to our list of test cases if (testCaseNumber != 1) { return(MochaParserStates.ExpectingTestResults); } // If the number was 1 then there's a good chance this is the beginning of the next test run, hence reset and start over // This is something we might choose to change if we realize there is a chance we can get such false detections often in the middle of a run AttemptPublishAndResetParser(); } // Increment either ways whether it was expected or context was reset and the encountered number was 1 mochaStateContext.LastFailedTestCaseNumber++; var testResult = PrepareTestResult(TestOutcome.Failed, match); mochaStateContext.TestRun.FailedTests.Add(testResult); return(MochaParserStates.ExpectingTestResults); }
private Enum StackTraceStartMatched(Match match, AbstractParserStateContext stateContext) { var jestStateContext = stateContext as JestParserStateContext; // Set this by default to -1, if a genuine stack trace was encountered then the actual index will be set. jestStateContext.CurrentStackTraceIndex = -1; if (jestStateContext.FailedTestsSummaryIndicatorEncountered) { Logger.Verbose($"{ParserName} : {StateName} : Ignoring StackTrace/Failed test case at line " + $"{stateContext.CurrentLineNumber} as it is part of the summarized failures."); return(JestParserStates.ExpectingStackTraces); } // In non verbose mode console out appears as a failed test case // Only difference being it's not colored red // Also this generally is the first "stack trace" hence this code is ideally // not likely to be hit but keeping it here as safety check if (match.Groups[RegexCaptureGroups.TestCaseName].Value == "Console") { Logger.Verbose($"{ParserName} : {StateName} : Ignoring apparent StackTrace/Failed test case at line " + $"{stateContext.CurrentLineNumber} as Jest prints console out in this format in non verbose mode."); return(JestParserStates.ExpectingStackTraces); } var testResult = PrepareTestResult(TestOutcome.Failed, match); jestStateContext.TestRun.FailedTests.Add(testResult); jestStateContext.CurrentStackTraceIndex = jestStateContext.TestRun.FailedTests.Count - 1; // Expect the stack trace to not be more than 50 lines long // This is to ensure we don't skip publishing the run if the stack traces appear corrupted jestStateContext.LinesWithinWhichMatchIsExpected = 50; jestStateContext.NextExpectedMatch = "next stacktraceStart/testrunStart/testrunSummary"; jestStateContext.TestRun.FailedTests[jestStateContext.CurrentStackTraceIndex].StackTrace = match.Value; return(JestParserStates.ExpectingStackTraces); }