public TestRunManagerTests() { this.publisher = new Mock <ITestRunPublisher>(); this.diagnosticDataCollector = new Mock <IDiagnosticDataCollector>(); this.telemetryDataCollector = new Mock <ITelemetryDataCollector>(); this.testRunManager = new TestRunManager(publisher.Object, diagnosticDataCollector.Object, telemetryDataCollector.Object); }
private void AddTestRun(object sender, RoutedEventArgs e) { string projectTitle = PromptDialog.Prompt("Test Run name", "Create new test run"); if (!string.IsNullOrWhiteSpace(projectTitle)) { TestRunManager runManager = new TestRunManager(); TestRunProxy proxyProject = ProxyConverter.TestRunModelToProxy(runManager.Create(projectTitle)); this.UITestRunList.Add(proxyProject); } }
public async Task TestRunManager_PublishTestRun_TestSummaryWithoutTests() { var logger = new Mock <ITraceLogger>(); var publisher = new Mock <ITestRunPublisher>(); var runManager = new TestRunManager(publisher.Object, logger.Object); publisher.Setup(x => x.PublishAsync(It.IsAny <TestRun>())).Returns(Task.CompletedTask); await runManager.PublishAsync(new TestRun("fake/1", 1)); publisher.Verify(x => x.PublishAsync(It.IsAny <TestRun>()), Times.Never()); logger.Verify(x => x.Error(It.IsAny <string>())); }
public async Task TestRunManager_PublishTestRun_TestSummaryWithoutTests() { var logger = new Mock <ITraceLogger>(); var publisher = new Mock <ITestRunPublisher>(); var telemetry = new Mock <ITelemetryDataCollector>(); var runManager = new TestRunManager(publisher.Object, logger.Object, telemetry.Object); publisher.Setup(x => x.PublishAsync(It.IsAny <TestRun>())); await runManager.PublishAsync(new TestRun("fake/1", "somename", 1)); publisher.Verify(x => x.PublishAsync(It.IsAny <TestRun>()), Times.Never()); logger.Verify(x => x.Error(It.IsAny <string>())); }
public async Task TestRunManager_PublishTestRun_TestRunIsNotValid() { var logger = new Mock <ITraceLogger>(); var publisher = new Mock <ITestRunPublisher>(); var telemetry = new Mock <ITelemetryDataCollector>(); var runManager = new TestRunManager(publisher.Object, logger.Object, telemetry.Object); publisher.Setup(x => x.PublishAsync(It.IsAny <TestRun>())); await runManager.PublishAsync(null); publisher.Verify(x => x.PublishAsync(It.IsAny <TestRun>()), Times.Never()); logger.Verify(x => x.Error(It.IsAny <string>())); }
/// <summary> /// Publish the current test run and reset the parser /// </summary> private void PublishAndReset(LogData logData) { Logger.Info($"PythonTestResultParser : PublishAndReset : Publishing TestRun {_currentTestRunId} at line {logData.LineNumber}."); foreach (var failedTest in _currentTestRun.FailedTests) { if (failedTest.StackTrace != null) { failedTest.StackTrace = failedTest.StackTrace.TrimEnd(); } } TestRunManager.PublishAsync(_currentTestRun); Reset(logData); }
/// <inheritdoc /> public async Task InitializeAsync(IClientFactory clientFactory, IPipelineConfig pipelineConfig, ITraceLogger traceLogger) { await Task.Run(() => { _logger = traceLogger; var publisher = new PipelineTestRunPublisher(clientFactory, pipelineConfig); var telemetry = new TelemetryDataCollector(clientFactory); _testRunManager = new TestRunManager(publisher, _logger); var parsers = ParserFactory.GetTestResultParsers(_testRunManager, traceLogger, telemetry); foreach (var parser in parsers) { //Subscribe parsers to Pub-Sub model Subscribe(parser.Parse); } }); }
public async Task TestRunManager_PublishMultipleRuns() { var logger = new Mock <ITraceLogger>(); var publisher = new Mock <ITestRunPublisher>(); var runManager = new TestRunManager(publisher.Object, logger.Object); var fakeRun = new TestRun("mocha/1", 1) { TestRunSummary = new TestRunSummary { TotalTests = 7 } }; publisher.SetupSequence(x => x.PublishAsync(It.IsAny <TestRun>())).Returns(Task.CompletedTask).Returns(Task.CompletedTask); RunTasks(runManager, fakeRun); await runManager.FinalizeAsync(); publisher.Verify(x => x.PublishAsync(It.IsAny <TestRun>()), Times.Exactly(2)); }
public async Task TestRunManager_PublishMultipleRuns() { var logger = new Mock <ITraceLogger>(); var publisher = new Mock <ITestRunPublisher>(); var telemetry = new Mock <ITelemetryDataCollector>(); var runManager = new TestRunManager(publisher.Object, logger.Object, telemetry.Object); var fakeRun = new TestRun("mocha/1", "somename", 1) { TestRunSummary = new TestRunSummary { TotalTests = 7 } }; publisher.SetupSequence(x => x.PublishAsync(It.IsAny <TestRun>())).Returns(Task.FromResult(GetFakePipelineTestRun(fakeRun, 1))).Returns(Task.FromResult(GetFakePipelineTestRun(fakeRun, 2))); RunTasks(runManager, fakeRun); await runManager.FinalizeAsync(); publisher.Verify(x => x.PublishAsync(It.IsAny <TestRun>()), Times.Exactly(2)); }
public async Task TestRunManager_PublishTestRun_SummaryDoesnotMatchTestRun() { var logger = new Mock <ITraceLogger>(); var publisher = new Mock <ITestRunPublisher>(); var telemetry = new Mock <ITelemetryDataCollector>(); var runManager = new TestRunManager(publisher.Object, logger.Object, telemetry.Object); var fakeRun = new TestRun("fake/1", "somename", 1) { TestRunSummary = new TestRunSummary { TotalPassed = 5, TotalFailed = 3, TotalSkipped = 2 }, PassedTests = new List <TestResult> { new TestResult() }, FailedTests = new List <TestResult> { new TestResult() }, SkippedTests = new List <TestResult> { new TestResult() } }; publisher.Setup(x => x.PublishAsync(It.IsAny <TestRun>())).Returns(Task.FromResult(GetFakePipelineTestRun(fakeRun, 1))); await runManager.PublishAsync(fakeRun); publisher.Verify(x => x.PublishAsync(It.Is <TestRun>(run => run.TestRunSummary.TotalTests == 10 && run.PassedTests.Count == 0 && run.FailedTests.Count == 0 && run.SkippedTests.Count == 0 && run.TestRunSummary.TotalPassed == 5 && run.TestRunSummary.TotalFailed == 3 && run.TestRunSummary.TotalSkipped == 2))); }
public static TestRunProxy TestRunModelToProxy(TestRun run) { TestRunProxy runProxy = new TestRunProxy(); runProxy.ID = run.ID; runProxy.Name = run.Name; runProxy.CreatedBy = run.CreatedBy; runProxy.CreatedOn = run.CreatedOn; IEnumerable<TestComposite> compositeModel = new TestRunManager().GetCompositeByRunId(runProxy.ID); foreach (TestComposite comp in compositeModel) { ExtendedTestCaseProxy extendedTestCaseProxy = new ExtendedTestCaseProxy(); extendedTestCaseProxy.Status = EnumUtil.ParseEnum<Status>(comp.TestCaseStatus); TestCase testCase = new TestManager().GetById(comp.TestCaseID); extendedTestCaseProxy.Id = testCase.ID; extendedTestCaseProxy.Title = testCase.Title; extendedTestCaseProxy.Priority = EnumUtil.ParseEnum<Priority>(testCase.Priority); extendedTestCaseProxy.Severity = EnumUtil.ParseEnum<Severity>(testCase.Severity); extendedTestCaseProxy.IsAutomated = testCase.IsAutomated; extendedTestCaseProxy.CreatedBy = testCase.CreatedBy; extendedTestCaseProxy.UpdatedBy = testCase.UpdatedBy; extendedTestCaseProxy.AreaID = testCase.AreaID; foreach (var item in new TestManager().GetStepDefinitionsById(testCase.ID)) { StepDefinitionProxy proxy = new StepDefinitionProxy(); proxy.Step = item.Step; proxy.ExpectedResult = item.ExpectedResult; proxy.ID = item.ID; proxy.TestCaseID = item.TestCaseID; extendedTestCaseProxy.StepDefinitionList.Add(proxy); } runProxy.TestCasesList.Add(extendedTestCaseProxy); } return runProxy; }
public async Task TestRunManager_PublishTestRun_TotalTestsLessThanActual() { var logger = new Mock <ITraceLogger>(); var publisher = new Mock <ITestRunPublisher>(); var runManager = new TestRunManager(publisher.Object, logger.Object); publisher.Setup(x => x.PublishAsync(It.IsAny <TestRun>())).Returns(Task.CompletedTask); await runManager.PublishAsync(new TestRun("fake/1", 1) { TestRunSummary = new TestRunSummary { TotalPassed = 5, TotalSkipped = 1, TotalFailed = 1, TotalExecutionTime = TimeSpan.FromMinutes(1), TotalTests = 6 } }); publisher.Verify(x => x.PublishAsync(It.Is <TestRun>(run => run.TestRunSummary.TotalTests == 7))); }
public async Task TestRunManager_PublishTestRun_SummaryDoesnotMatchTestRun() { var logger = new Mock <ITraceLogger>(); var publisher = new Mock <ITestRunPublisher>(); var runManager = new TestRunManager(publisher.Object, logger.Object); publisher.Setup(x => x.PublishAsync(It.IsAny <TestRun>())).Returns(Task.CompletedTask); await runManager.PublishAsync(new TestRun("fake/1", 1) { TestRunSummary = new TestRunSummary { TotalPassed = 5, TotalFailed = 3, TotalSkipped = 2 }, PassedTests = new List <TestResult> { new TestResult() }, FailedTests = new List <TestResult> { new TestResult() }, SkippedTests = new List <TestResult> { new TestResult() } }); publisher.Verify(x => x.PublishAsync(It.Is <TestRun>(run => run.TestRunSummary.TotalTests == 10 && run.PassedTests.Count == 0 && run.FailedTests.Count == 0 && run.SkippedTests.Count == 0 && run.TestRunSummary.TotalPassed == 5 && run.TestRunSummary.TotalFailed == 3 && run.TestRunSummary.TotalSkipped == 2))); }
public async Task TestRunManager_PublishTestRun() { var logger = new Mock <ITraceLogger>(); var publisher = new Mock <ITestRunPublisher>(); var runManager = new TestRunManager(publisher.Object, logger.Object); var fakeRun = new TestRun("mocha/1", 1) { TestRunSummary = new TestRunSummary { TotalPassed = 5, TotalSkipped = 1, TotalFailed = 1, TotalExecutionTime = TimeSpan.FromMinutes(1), TotalTests = 7 } }; publisher.Setup(x => x.PublishAsync(It.IsAny <TestRun>())).Returns(Task.CompletedTask); await runManager.PublishAsync(fakeRun); publisher.Verify(x => x.PublishAsync(It.IsAny <TestRun>())); }
public async Task TestRunManager_PublishTestRun() { var logger = new Mock <ITraceLogger>(); var publisher = new Mock <ITestRunPublisher>(); var telemetry = new Mock <ITelemetryDataCollector>(); var runManager = new TestRunManager(publisher.Object, logger.Object, telemetry.Object); var fakeRun = new TestRun("mocha/1", "somename", 1) { TestRunSummary = new TestRunSummary { TotalPassed = 5, TotalSkipped = 1, TotalFailed = 1, TotalExecutionTime = TimeSpan.FromMinutes(1), TotalTests = 7 } }; publisher.Setup(x => x.PublishAsync(It.IsAny <TestRun>())).Returns(Task.FromResult(GetFakePipelineTestRun(fakeRun, 1))); await runManager.PublishAsync(fakeRun); publisher.Verify(x => x.PublishAsync(It.IsAny <TestRun>())); }
private void SaveRunStatus_Click(object sender, RoutedEventArgs e) { this.SetPreviousTestCaseStatus(); foreach (KeyValuePair<ExtendedTestCaseProxy, Status> item in runStatus) { TestRunManager manager = new TestRunManager(); manager.UpdateTestCaseStatus(RunId, item.Key.Id, item.Value); } this.CancelDialog(); }
/// <summary> /// Publishes the run and resets the parser by resetting the state context and current state /// </summary> private void AttemptPublishAndResetParser() { Logger.Info($"MochaTestResultParser : Resetting the parser and attempting to publish the test run at line {_stateContext.CurrentLineNumber}."); var testRunToPublish = _stateContext.TestRun; // We have encountered failed test cases but no failed summary was encountered if (testRunToPublish.FailedTests.Count != 0 && testRunToPublish.TestRunSummary.TotalFailed == 0) { Logger.Error("MochaTestResultParser : Failed tests were encountered but no failed summary was encountered."); Telemetry.AddAndAggregate(MochaTelemetryConstants.FailedTestCasesFoundButNoFailedSummary, new List <int> { _stateContext.TestRun.TestRunId }, MochaTelemetryConstants.EventArea); } else if (testRunToPublish.TestRunSummary.TotalFailed != testRunToPublish.FailedTests.Count) { // If encountered failed tests does not match summary fire telemetry Logger.Error($"MochaTestResultParser : Failed tests count does not match failed summary" + $" at line {_stateContext.CurrentLineNumber}"); Telemetry.AddAndAggregate(MochaTelemetryConstants.FailedSummaryMismatch, new List <int> { testRunToPublish.TestRunId }, MochaTelemetryConstants.EventArea); } // We have encountered pending test cases but no pending summary was encountered if (testRunToPublish.SkippedTests.Count != 0 && testRunToPublish.TestRunSummary.TotalSkipped == 0) { Logger.Error("MochaTestResultParser : Skipped tests were encountered but no skipped summary was encountered."); Telemetry.AddAndAggregate(MochaTelemetryConstants.PendingTestCasesFoundButNoFailedSummary, new List <int> { _stateContext.TestRun.TestRunId }, MochaTelemetryConstants.EventArea); } else if (testRunToPublish.TestRunSummary.TotalSkipped != testRunToPublish.SkippedTests.Count) { // If encountered skipped tests does not match summary fire telemetry Logger.Error($"MochaTestResultParser : Pending tests count does not match pending summary" + $" at line {_stateContext.CurrentLineNumber}"); Telemetry.AddAndAggregate(MochaTelemetryConstants.PendingSummaryMismatch, new List <int> { testRunToPublish.TestRunId }, MochaTelemetryConstants.EventArea); } // Ensure some summary data was detected before attempting a publish, ie. check if the state is not test results state switch (_currentState) { case MochaParserStates.ExpectingTestResults: if (testRunToPublish.PassedTests.Count != 0 || testRunToPublish.FailedTests.Count != 0 || testRunToPublish.SkippedTests.Count != 0) { Logger.Error("MochaTestResultParser : Skipping publish as testcases were encountered but no summary was encountered."); Telemetry.AddAndAggregate(MochaTelemetryConstants.PassedTestCasesFoundButNoPassedSummary, new List <int> { _stateContext.TestRun.TestRunId }, MochaTelemetryConstants.EventArea); } break; default: // Publish the test run if reset and publish was called from any state other than the test results state // Calculate total tests testRunToPublish.TestRunSummary.TotalTests = testRunToPublish.TestRunSummary.TotalPassed + testRunToPublish.TestRunSummary.TotalFailed + testRunToPublish.TestRunSummary.TotalSkipped; // Trim the stack traces of extra newlines etc. foreach (var failedTest in testRunToPublish.FailedTests) { if (failedTest.StackTrace != null) { failedTest.StackTrace = failedTest.StackTrace.TrimEnd(); } } TestRunManager.PublishAsync(testRunToPublish); break; } ResetParser(); }
/// <summary> /// Publishes the run and resets the parser by resetting the state context and current state /// </summary> private void AttemptPublishAndResetParser() { Logger.Info($"JestTestResultParser : Resetting the parser and attempting to publish the test run at line {_stateContext.CurrentLineNumber}."); var testRunToPublish = _stateContext.TestRun; // We have encountered passed test cases but no passed summary was encountered if (testRunToPublish.PassedTests.Count != 0 && testRunToPublish.TestRunSummary.TotalPassed == 0) { Logger.Error("JestTestResultParser : Passed tests were encountered but no passed summary was encountered."); Telemetry.AddAndAggregate(JestTelemetryConstants.PassedTestCasesFoundButNoPassedSummary, new List <int> { _stateContext.TestRun.TestRunId }, JestTelemetryConstants.EventArea); } else if (_stateContext.VerboseOptionEnabled && testRunToPublish.TestRunSummary.TotalPassed != testRunToPublish.PassedTests.Count) { // If encountered failed tests does not match summary fire telemetry Logger.Error($"JestTestResultParser : Passed tests count does not match passed summary" + $" at line {_stateContext.CurrentLineNumber}"); Telemetry.AddAndAggregate(JestTelemetryConstants.PassedSummaryMismatch, new List <int> { testRunToPublish.TestRunId }, JestTelemetryConstants.EventArea); } // We have encountered failed test cases but no failed summary was encountered if (testRunToPublish.FailedTests.Count != 0 && testRunToPublish.TestRunSummary.TotalFailed == 0) { Logger.Error("JestTestResultParser : Failed tests were encountered but no failed summary was encountered."); Telemetry.AddAndAggregate(JestTelemetryConstants.FailedTestCasesFoundButNoFailedSummary, new List <int> { _stateContext.TestRun.TestRunId }, JestTelemetryConstants.EventArea); } else if (testRunToPublish.TestRunSummary.TotalFailed != testRunToPublish.FailedTests.Count) { // If encountered failed tests does not match summary fire telemtry Logger.Error($"JestTestResultParser : Failed tests count does not match failed summary" + $" at line {_stateContext.CurrentLineNumber}"); Telemetry.AddAndAggregate(JestTelemetryConstants.FailedSummaryMismatch, new List <int> { testRunToPublish.TestRunId }, JestTelemetryConstants.EventArea); } // Ensure some summary data was detected before attempting a publish, ie. check if the state is not test results state switch (_currentState) { case JestParserStates.ExpectingTestRunStart: Logger.Error("JestTestResultParser : Skipping publish as no test cases or summary has been encountered."); break; case JestParserStates.ExpectingTestResults: case JestParserStates.ExpectingStackTraces: if (testRunToPublish.PassedTests.Count != 0 || testRunToPublish.FailedTests.Count != 0 || testRunToPublish.SkippedTests.Count != 0) { Logger.Error("JestTestResultParser : Skipping publish as testcases were encountered but no summary was encountered."); Telemetry.AddAndAggregate(JestTelemetryConstants.TestCasesFoundButNoSummary, new List <int> { _stateContext.TestRun.TestRunId }, JestTelemetryConstants.EventArea); } break; case JestParserStates.ExpectingTestRunSummary: if (testRunToPublish.TestRunSummary.TotalTests == 0) { Logger.Error("JestTestResultParser : Skipping publish as total tests was 0."); Telemetry.AddAndAggregate(JestTelemetryConstants.TotalTestsZero, new List <int> { _stateContext.TestRun.TestRunId }, JestTelemetryConstants.EventArea); break; } if (testRunToPublish.TestRunSummary.TotalExecutionTime.TotalMilliseconds == 0) { Logger.Error("JestTestResultParser : Total test run time was 0 or not encountered."); Telemetry.AddAndAggregate(JestTelemetryConstants.TotalTestRunTimeZero, new List <int> { _stateContext.TestRun.TestRunId }, JestTelemetryConstants.EventArea); } // Trim the stack traces of extra newlines etc. foreach (var failedTest in testRunToPublish.FailedTests) { if (failedTest.StackTrace != null) { failedTest.StackTrace = failedTest.StackTrace.TrimEnd(); } } // Only publish if total tests was not zero TestRunManager.PublishAsync(testRunToPublish); break; } ResetParser(); }
private void SaveRun(object sender, RoutedEventArgs e) { ICollection<TestCase> testCasesModelList = new Collection<TestCase>(); this.TestCasesList.ToList().ForEach(x => { testCasesModelList.Add(ModelConverter.TestCaseProxyToModel(x)); }); TestRunManager manager = new TestRunManager(); manager.RelateTestCaseToTestRun(this.TestRunProxy.ID, testCasesModelList); this.CancelDialog(); }
public void TestInitialize() => TestRunManager = new TestRunManager(TestContext.TestName);