private void ExecuteTestsWithTestRunner( IEnumerable <TestCase> tests, IRunContext runContext, ITestExecutionRecorder testExecutionRecorder, string source, IDictionary <string, object> sourceLevelParameters, UnitTestRunner testRunner) { foreach (var currentTest in tests) { if (this.cancellationToken != null && this.cancellationToken.Canceled) { break; } var unitTestElement = currentTest.ToUnitTestElement(source); testExecutionRecorder.RecordStart(currentTest); var startTime = DateTimeOffset.Now; PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Executing test {0}", unitTestElement.TestMethod.Name); // Run single test passing test context properties to it. var tcmProperties = TcmTestPropertiesProvider.GetTcmProperties(currentTest); var testContextProperties = this.GetTestContextProperties(tcmProperties, sourceLevelParameters); var unitTestResult = testRunner.RunSingleTest(unitTestElement.TestMethod, testContextProperties); PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Executed test {0}", unitTestElement.TestMethod.Name); var endTime = DateTimeOffset.Now; this.SendTestResults(currentTest, unitTestResult, startTime, endTime, testExecutionRecorder); } }
private void RunTest(ITestExecutionRecorder frameworkHandle, string source, string spec = null) { var results = appDomainRunner.ExecuteSpecifications(source, spec); var query = from result in results from @group in result.Examples from example in @group.Examples select example; foreach (var example in query) { var testCase = new TestCase(example.Reason, ExecutorUri, source) { CodeFilePath = source }; frameworkHandle.RecordStart(testCase); var testResult = new TestResult(testCase) { DisplayName = example.Reason, Duration = new TimeSpan(example.ElapsedTime), }; if (example.Status == ResultStatus.Error) { testResult.Outcome = TestOutcome.Failed; testResult.ErrorMessage = example.Message; testResult.ErrorStackTrace = example.StackTrace; } if (example.Status == ResultStatus.Success) { testResult.Outcome = TestOutcome.Passed; } frameworkHandle.RecordEnd(testCase, testResult.Outcome); frameworkHandle.RecordResult(testResult); } }
private bool Deploy(string source, IRunContext runContext, ITestExecutionRecorder testExecutionRecorder, IList <DeploymentItem> deploymentItems, ref TestRunDirectories testRunDirectories) { if (EqtTrace.IsInfoEnabled) { EqtTrace.Info("MSTestExecutor: Found that deployment items for source {0} are: ", source); foreach (var item in deploymentItems) { EqtTrace.Info("MSTestExecutor: SourcePath: - {0}", item.SourcePath); } } // Do the deployment. IEnumerable <string> warnings; var runDirectories = testRunDirectories ?? this.CreateDeploymentDirectories(runContext); ValidateArg.NotNull(runDirectories, "runDirectories"); EqtTrace.InfoIf(EqtTrace.IsInfoEnabled, "MSTestExecutor: Using deployment directory {0} for source {1}.", runDirectories.OutDirectory, source); this.Deploy(new List <DeploymentItem>(deploymentItems), source, runDirectories.OutDirectory, out warnings); // Log warnings LogWarnings(testExecutionRecorder, warnings); return(deploymentItems != null && deploymentItems.Count > 0); }
public VsExecutionSink(IMessageSinkWithTypes innerSink, ITestExecutionRecorder recorder, LoggerHelper logger, Dictionary <string, TestCase> testCasesMap, ITestFrameworkExecutionOptions executionOptions, Func <bool> cancelledThunk) { this.innerSink = innerSink; this.recorder = recorder; this.logger = logger; this.testCasesMap = testCasesMap; this.executionOptions = executionOptions; this.cancelledThunk = cancelledThunk; ExecutionSummary = new ExecutionSummary(); Diagnostics.ErrorMessageEvent += HandleErrorMessage; Execution.TestAssemblyCleanupFailureEvent += HandleTestAssemblyCleanupFailure; Execution.TestAssemblyFinishedEvent += HandleTestAssemblyFinished; Execution.TestCaseCleanupFailureEvent += HandleTestCaseCleanupFailure; Execution.TestCaseFinishedEvent += HandleTestCaseFinished; Execution.TestCaseStartingEvent += HandleTestCaseStarting; Execution.TestClassCleanupFailureEvent += HandleTestClassCleanupFailure; Execution.TestCleanupFailureEvent += HandleTestCleanupFailure; Execution.TestCollectionCleanupFailureEvent += HandleTestCollectionCleanupFailure; Execution.TestFailedEvent += HandleTestFailed; Execution.TestMethodCleanupFailureEvent += HandleTestMethodCleanupFailure; Execution.TestPassedEvent += HandleTestPassed; Execution.TestSkippedEvent += HandleTestSkipped; }
/// <summary> /// Log the parameter warnings on the parameter logger /// </summary> /// <param name="testExecutionRecorder">Handle to record test start/end/results/messages.</param> /// <param name="result">Result of the run operation.</param> private void LogCleanupResult(ITestExecutionRecorder testExecutionRecorder, RunCleanupResult result) { Debug.Assert(testExecutionRecorder != null, "Logger should not be null"); if (!string.IsNullOrEmpty(result.StandardOut)) { testExecutionRecorder.SendMessage(TestMessageLevel.Informational, result.StandardOut); } if (!string.IsNullOrEmpty(result.DebugTrace)) { testExecutionRecorder.SendMessage(TestMessageLevel.Informational, result.DebugTrace); } if (!string.IsNullOrEmpty(result.StandardError)) { testExecutionRecorder.SendMessage( MSTestSettings.CurrentSettings.TreatClassAndAssemblyCleanupWarningsAsErrors ? TestMessageLevel.Error : TestMessageLevel.Warning, result.StandardError); } if (result.Warnings != null) { foreach (string warning in result.Warnings) { testExecutionRecorder.SendMessage( MSTestSettings.CurrentSettings.TreatClassAndAssemblyCleanupWarningsAsErrors ? TestMessageLevel.Error : TestMessageLevel.Warning, warning); } } }
public TestRunner(IEnumerable <string> sources, IRunSettings runSettings, ITestExecutionRecorder recorder) { RunSettings = RunSettings; sink = new TestCaseDiscoverySink(); new MSTestDiscoverer().DiscoverTests(sources, this, this, sink); this.recorder = recorder; }
public NUnitEventListener(ITestConverterCommon testConverter, INUnit3TestExecutor executor) { this.executor = executor; dumpXml = executor.Dump; _settings = executor.Settings; _recorder = executor.FrameworkHandle; _testConverter = testConverter; }
public NUnitEventListener(ITestExecutionRecorder recorder, ITestConverter testConverter, INUnit3TestExecutor executor) { this.executor = executor; dumpXml = executor.Dump; _settings = executor.Settings; _recorder = recorder; _testConverter = testConverter; }
public virtual void before_each() { autoSubstitute = new AutoSubstitute(); testExecutionRecorder = autoSubstitute.Resolve <ITestExecutionRecorder>(); recorder = autoSubstitute.Resolve <ProgressRecorder>(); }
public void Setup() { recorder = Substitute.For <IFrameworkHandle>(); converter = Substitute.For <ITestConverterCommon>(); settings = Substitute.For <IAdapterSettings>(); executor = Substitute.For <INUnit3TestExecutor>(); executor.Settings.Returns(settings); executor.FrameworkHandle.Returns(recorder); }
public void Setup() { recorder = Substitute.For <ITestExecutionRecorder>(); converter = Substitute.For <ITestConverter>(); dumpxml = Substitute.For <IDumpXml>(); settings = Substitute.For <IAdapterSettings>(); executor = Substitute.For <INUnit3TestExecutor>(); executor.Settings.Returns(settings); }
public VsExecutionVisitor(ITestExecutionRecorder recorder, Dictionary <ITestCase, TestCase> testCases, ITestFrameworkExecutionOptions executionOptions, Func <bool> cancelledThunk) { this.recorder = recorder; this.testCases = testCases; this.executionOptions = executionOptions; this.cancelledThunk = cancelledThunk; }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, ITestExecutionRecorder testExecutionRecorder) { Guard.ArgumentNotNull("sources", sources); Guard.ArgumentNotNull("runContext", runContext); Guard.ArgumentNotNull("testExecutionRecorder", testExecutionRecorder); foreach (string source in sources) if (VsTestRunner.IsXunitTestAssembly(source)) RunTestsInAssembly(source, runContext, testExecutionRecorder); }
public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, ITestExecutionRecorder testExecutionRecorder) { Guard.ArgumentNotNull("tests", tests); Guard.ArgumentNotNull("runContext", runContext); Guard.ArgumentNotNull("testExecutionRecorder", testExecutionRecorder); foreach (var testCaseGroup in tests.GroupBy(tc => tc.Source)) if (VsTestRunner.IsXunitTestAssembly(testCaseGroup.Key)) RunTestsInAssembly(testCaseGroup.Key, runContext, testExecutionRecorder, testCaseGroup); }
public void RunTests(IEnumerable <string> sources, IRunContext runContext, ITestExecutionRecorder testExecutionRecorder) { _launcher = new TestLauncher(); foreach (string source in sources) { _launcher.AddFilePattern(source); } RunTests(testExecutionRecorder); }
public void RunTests(IEnumerable <string> sources, IRunContext runContext, ITestExecutionRecorder testExecutionRecorder) { Guard.ArgumentNotNull("sources", sources); Guard.ArgumentNotNull("runContext", runContext); Guard.ArgumentNotNull("testExecutionRecorder", testExecutionRecorder); foreach (string source in sources) { if (VsTestRunner.IsXunitTestAssembly(source)) { RunTestsInAssembly(source, runContext, testExecutionRecorder); } } }
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, ITestExecutionRecorder testExecutionRecorder) { Guard.ArgumentNotNull("tests", tests); Guard.ArgumentNotNull("runContext", runContext); Guard.ArgumentNotNull("testExecutionRecorder", testExecutionRecorder); foreach (var testCaseGroup in tests.GroupBy(tc => tc.Source)) { if (VsTestRunner.IsXunitTestAssembly(testCaseGroup.Key)) { RunTestsInAssembly(testCaseGroup.Key, runContext, testExecutionRecorder, testCaseGroup); } } }
public VsExecutionVisitor(ITestExecutionRecorder recorder, LoggerHelper logger, Dictionary <ITestCase, TestCase> testCases, ITestFrameworkExecutionOptions executionOptions, Func <bool> cancelledThunk) { this.recorder = recorder; this.logger = logger; this.testCases = testCases; this.executionOptions = executionOptions; this.cancelledThunk = cancelledThunk; ExecutionSummary = new ExecutionSummary(); }
//not picking up public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, ITestExecutionRecorder testExecutionRecorder) { _launcher = new TestLauncher(); _launcher.RuntimeSetup = new RuntimeSetup(); //_launcher.TestProject. foreach (var test in tests) { _launcher.AddFilePattern(test.Source); _launcher.TestExecutionOptions.FilterSet = FilterUtils.ParseTestFilterSet("ExactType:" + test.DisplayName); } RunTests(testExecutionRecorder); }
public VsExecutionVisitor(IDiscoveryContext discoveryContext, ITestExecutionRecorder recorder, Dictionary <ITestCase, TestCase> testCases, Func <bool> cancelledThunk) { this.recorder = recorder; this.testCases = testCases; this.cancelledThunk = cancelledThunk; settings = SettingsProvider.Load(); var settingsProvider = discoveryContext.RunSettings.GetSettings(XunitTestRunSettings.SettingsName) as XunitTestRunSettingsProvider; if (settingsProvider != null && settingsProvider.Settings != null) { settings.Merge(settingsProvider.Settings); } }
/// <summary> /// Log the parameter warnings on the parameter logger /// </summary> /// <param name="testExecutionRecorder">Execution recorder.</param> /// <param name="warnings">Warnings.</param> private static void LogWarnings(ITestExecutionRecorder testExecutionRecorder, IEnumerable <string> warnings) { if (warnings == null) { return; } Debug.Assert(testExecutionRecorder != null, "Logger should not be null"); // log the warnings foreach (string warning in warnings) { testExecutionRecorder.SendMessage(TestMessageLevel.Warning, warning); } }
private void RunCleanup( ITestExecutionRecorder testExecutionRecorder, UnitTestRunner testRunner) { // All cleanups (Class and Assembly) run at the end of test execution. Failures in these cleanup // methods will be reported as Warnings. PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Executing cleanup methods."); var cleanupResult = testRunner.RunCleanup(); PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Executed cleanup methods."); if (cleanupResult != null) { // Do not attach the standard output and error messages to any test result. It is not // guaranteed that a test method of same class would have run last. We will end up // adding stdout to test method of another class. this.LogCleanupResult(testExecutionRecorder, cleanupResult); } }
private void ExecuteTestsWithTestRunner( IEnumerable <TestCase> tests, IRunContext runContext, ITestExecutionRecorder testExecutionRecorder, string source, IDictionary <string, object> sourceLevelParameters, UnitTestRunner testRunner) { var testExecutionRecorderWrapper = new TestExecutionRecorderWrapper(testExecutionRecorder); foreach (var currentTest in tests) { if (this.cancellationToken != null && this.cancellationToken.Canceled) { break; } var unitTestElement = currentTest.ToUnitTestElement(source); // Fire a RecordStart here even though we also fire RecordStart inside TestMethodInfo // this is to ensure we fire start events for error cases. This is acceptable because // vstest ignores multiple start events. testExecutionRecorder.RecordStart(currentTest); var startTime = DateTimeOffset.Now; PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo( "Executing test {0}", unitTestElement.TestMethod.Name); // Run single test passing test context properties to it. var tcmProperties = TcmTestPropertiesProvider.GetTcmProperties(currentTest); var testContextProperties = this.GetTestContextProperties(tcmProperties, sourceLevelParameters); var unitTestResult = testRunner.RunSingleTest(unitTestElement.TestMethod, testExecutionRecorderWrapper, testContextProperties); PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo( "Executed test {0}", unitTestElement.TestMethod.Name); var endTime = DateTimeOffset.Now; this.SendTestResults(currentTest, unitTestResult, startTime, endTime, testExecutionRecorder); } }
void RunTestsInAssembly(string assemblyFileName, IRunContext ctxt, ITestExecutionRecorder recorder, IEnumerable <TestCase> testCases = null) { cancelled = false; using (var executor = new ExecutorWrapper(assemblyFileName, configFilename: null, shadowCopy: true)) { if (testCases == null) { testCases = VsTestRunner.GetTestCases(executor).ToArray(); } var logger = new VsRunnerLogger(recorder, testCases, () => cancelled); var runner = new TestRunner(executor, logger); foreach (var testClass in testCases.Select(tc => new TypeAndMethod(tc.Name)) .GroupBy(tam => tam.Type)) { runner.RunTests(testClass.Key, testClass.Select(tam => tam.Method).ToList()); } } }
private void RunTests(ITestExecutionRecorder testExecutionRecorder) { TestLauncherResult testLauncherResult = _launcher.Run(); foreach (var allTestStepRun in testLauncherResult.Report.TestPackageRun.AllTestStepRuns) { TestCase tc = new TestCase(allTestStepRun.Step.FullName, new Uri(ExecutorUri)); ObjectModel.TestResult testResult = new ObjectModel.TestResult(tc); testResult.DisplayName = allTestStepRun.Step.FullName; testResult.ErrorLineNumber = allTestStepRun.Step.CodeLocation.Line; //testResult.ErrorStackTrace testResult.StartTime = allTestStepRun.StartTime; //testResult.ErrorMessage -- why this and messages? testResult.EndTime = allTestStepRun.EndTime; //testResult.Attachments = allTestStepRun.TestLog.Attachments; -- can I append my own? testResult.Duration = allTestStepRun.Result.Duration; //testResult.Messages = var testStatus = allTestStepRun.Result.Outcome.Status; switch (testStatus) { case TestStatus.Passed: testResult.Outcome = ObjectModel.TestOutcome.Passed; break; case TestStatus.Failed: testResult.Outcome = ObjectModel.TestOutcome.Failed; break; case TestStatus.Skipped: testResult.Outcome = ObjectModel.TestOutcome.Skipped; break; case TestStatus.Inconclusive: //is this right to use not found? testResult.Outcome = ObjectModel.TestOutcome.Skipped; break; } testExecutionRecorder.RecordResult(testResult); } }
private void CacheSessionParameters(IRunContext runContext, ITestExecutionRecorder testExecutionRecorder) { if (!string.IsNullOrEmpty(runContext?.RunSettings?.SettingsXml)) { try { var testRunParameters = RunSettingsUtilities.GetTestRunParameters(runContext.RunSettings.SettingsXml); if (testRunParameters != null) { foreach (var kvp in testRunParameters) { this.sessionParameters.Add(kvp); } } } catch (Exception ex) { testExecutionRecorder.SendMessage(TestMessageLevel.Error, ex.Message); } } }
//---------------------------------------------------------------------------------------------------- private void RunAndParseTests( ITestExecutionRecorder recorder, string executable, IEnumerable <TestCase> tests ) { var args = tests.Select(t => t.FullyQualifiedName).ToList(); args.Insert(0, "--verbose --adapter_info"); Task parse_output_task; lock (this) { if (m_test_run_process != null) { LogInfo("Cannot run tests while other tests are active"); return; } m_test_run_process = StartTestRun(executable, args.ToArray()); if (m_test_run_process == null) { LogError("Failed to launch " + executable); } parse_output_task = ParseTestRunOutput(recorder, m_test_run_process.StandardOutput, tests); } parse_output_task.Wait(); if (parse_output_task.Status != TaskStatus.RanToCompletion) { LogError("Failed to discover tests for " + executable); } lock (this) { m_test_run_process = null; } }
private void RunTest(ITestExecutionRecorder frameworkHandle, string source, string spec = null) { var results = appDomainRunner.ExecuteSpecifications(source, spec); var query = from result in results from @group in result.Examples from example in @group.Examples select example; foreach (var example in query) { var testCase = new TestCase(example.Reason, ExecutorUri, source) { CodeFilePath = source }; frameworkHandle.RecordStart(testCase); var testResult = new TestResult(testCase) { DisplayName = example.Reason, Duration = new TimeSpan(example.ElapsedTime), }; if (example.Status == ResultStatus.Error) { testResult.Outcome = TestOutcome.Failed; testResult.ErrorMessage = example.Message; testResult.ErrorStackTrace = example.StackTrace; } if (example.Status == ResultStatus.Success) { testResult.Outcome = TestOutcome.Passed; } frameworkHandle.RecordEnd(testCase, testResult.Outcome); frameworkHandle.RecordResult(testResult); } }
private Task ExecuteTests(IEnumerable <TestCase> tests, ITestExecutionRecorder frameworkHandle, Executor executor, CancellationToken token) { return(Task.WhenAll(tests.Select(test => Task.Run(async() => { frameworkHandle.RecordStart(test); try { var result = await executor.Execute(_testDiscoverer.GetTest(test.FullyQualifiedName), token); frameworkHandle.RecordEnd(test, result.Outcome.ToVsResultOutcome()); frameworkHandle.RecordResult(result.ConvertToVsResult(test)); } catch (Exception ex) { frameworkHandle.RecordEnd(test, TestOutcome.None); frameworkHandle.RecordResult(new TestResult(test) { Outcome = TestOutcome.None, ErrorMessage = ex.Message, ErrorStackTrace = ex.StackTrace }); } }, token)))); }
private void CacheSessionParameters(IRunContext runContext, ITestExecutionRecorder testExecutionRecorder) { if (!string.IsNullOrEmpty(runContext?.RunSettings?.SettingsXml)) { try { var testRunParameters = RunSettingsUtilities.GetTestRunParameters(runContext.RunSettings.SettingsXml); if (testRunParameters != null) { // Clear sessionParameters to prevent key collisions of test run parameters in case // "Keep Test Execution Engine Alive" is selected in VS. this.sessionParameters.Clear(); foreach (var kvp in testRunParameters) { this.sessionParameters.Add(kvp); } } } catch (Exception ex) { testExecutionRecorder.SendMessage(TestMessageLevel.Error, ex.Message); } } }
public VisualStudioListener(ITestExecutionRecorder log, string assemblyPath) { this.log = log; this.assemblyPath = assemblyPath; }
public NUnitEventListener(ITestExecutionRecorder testLog, TestConverter testConverter) { this.testLog = testLog; this.testConverter = testConverter; }
public VSTestWindowExtension(ITestExecutionRecorder executionRecorder, ITestCaseFactory testCaseFactory, ITestResultFactory testResultFactory) { this.executionRecorder = executionRecorder; this.testCaseFactory = testCaseFactory; this.testResultFactory = testResultFactory; }
public VisualStudioListener(ITestExecutionRecorder log, string assemblyPath) { this.log = log; this.assemblyPath = assemblyPath; }
void RunTestsInAssembly(string assemblyFileName, IRunContext ctxt, ITestExecutionRecorder recorder, IEnumerable<TestCase> testCases = null) { cancelled = false; using (var executor = new ExecutorWrapper(assemblyFileName, configFilename: null, shadowCopy: true)) { if (testCases == null) testCases = VsTestRunner.GetTestCases(executor).ToArray(); var logger = new VsRunnerLogger(recorder, testCases, () => cancelled); var runner = new TestRunner(executor, logger); foreach (var testClass in testCases.Select(tc => new TypeAndMethod(tc.Name)) .GroupBy(tam => tam.Type)) runner.RunTests(testClass.Key, testClass.Select(tam => tam.Method).ToList()); } }
public ProgressMonitorPresenter(ITestExecutionRecorder executionRecorder) { _executionRecorder = executionRecorder; }
public bool Deploy(IEnumerable <TestCase> tests, string source, IRunContext runContext, ITestExecutionRecorder testExecutionRecorder, TestRunDirectories runDirectories) { IList <DeploymentItem> deploymentItems = this.DeploymentItemUtility.GetDeploymentItems(tests); // we just deploy source if there are no deployment items for current source but there are deployment items for other sources return(this.Deploy(source, runContext, testExecutionRecorder, deploymentItems, runDirectories)); }
public VsRunnerLogger(ITestExecutionRecorder recorder, IEnumerable<TestCase> testCases, Func<bool> cancelledThunk) { this.recorder = recorder; this.testCases = testCases.ToDictionary(tc => tc.FullyQualifiedName); this.cancelledThunk = cancelledThunk; }
public ProgressRecorder(ITestExecutionRecorder testExecutionRecorder, ITestResultMapper testResultMapper) { this.testExecutionRecorder = testExecutionRecorder; this.testResultMapper = testResultMapper; }
public VSTestWindowExtension(ITestExecutionRecorder executionRecorder) { _executionRecorder = executionRecorder; }
void RunTestsInAssembly(List<ExecutorWrapper> cleanupList, string assemblyFileName, ITestExecutionRecorder recorder, IEnumerable<TestCase> testCases = null) { if (cancelled) return; var executor = new ExecutorWrapper(assemblyFileName, configFileName: null, shadowCopy: true); cleanupList.Add(executor); if (testCases == null) testCases = VsTestRunner.GetTestCases(executor).ToArray(); var logger = new VsRunnerLogger(recorder, testCases, () => cancelled); var runner = new TestRunner(executor, logger); foreach (var testClass in testCases.Select(tc => new TypeAndMethod(tc.FullyQualifiedName)) .GroupBy(tam => tam.Type)) { runner.RunTests(testClass.Key, testClass.Select(tam => tam.Method).ToList()); if (cancelled) return; } }
public NUnitEventListener(ITestExecutionRecorder recorder, TestConverter testConverter) { _recorder = recorder; _testConverter = testConverter; }