/// <summary> /// Runs the tests. /// </summary> /// <param name="tests">Which tests should be run.</param> /// <param name="context">Context in which to run tests.</param> /// <param param name="framework">Where results should be stored.</param> public void RunTests(IEnumerable<TestCase> tests, IRunContext context, IFrameworkHandle framework) { _state = ExecutorState.Running; foreach (var test in tests) { if (_state == ExecutorState.Cancelling) { _state = ExecutorState.Cancelled; return; } try { var reportDocument = RunOrDebugCatchTest(test.Source, test.FullyQualifiedName, context, framework); var result = GetTestResultFromReport(test, reportDocument, framework); framework.RecordResult(result); } catch (Exception ex) { // Log it and move on. It will show up to the user as a test that hasn't been run. framework.SendMessage(TestMessageLevel.Error, "Exception occured when processing test case: " + test.FullyQualifiedName); framework.SendMessage(TestMessageLevel.Informational, "Message: " + ex.Message + "\nStacktrace:" + ex.StackTrace); } } }
// NOTE: an earlier version of this code had a FilterBuilder // property. This seemed to make sense, because we instantiate // it in two different places. However, the existence of an // NUnitTestFilterBuilder, containing a reference to an engine // service caused our second-level tests of the test executor // to throw an exception. So if you consider doing this, beware! #endregion #region ITestExecutor Implementation /// <summary> /// Called by the Visual Studio IDE to run all tests. Also called by TFS Build /// to run either all or selected tests. In the latter case, a filter is provided /// as part of the run context. /// </summary> /// <param name="sources">Sources to be run.</param> /// <param name="runContext">Context to use when executing the tests.</param> /// <param name="frameworkHandle">Test log to send results and messages through</param> public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { #if LAUNCHDEBUGGER if (!Debugger.IsAttached) Debugger.Launch(); #endif Initialize(runContext, frameworkHandle); try { foreach (var source in sources) { var assemblyName = source; if (!Path.IsPathRooted(assemblyName)) assemblyName = Path.Combine(Environment.CurrentDirectory, assemblyName); TestLog.Info("Running all tests in " + assemblyName); RunAssembly(assemblyName, TestFilter.Empty); } } catch (Exception ex) { if (ex is TargetInvocationException) ex = ex.InnerException; TestLog.Error("Exception thrown executing tests", ex); } finally { TestLog.Info(string.Format("NUnit Adapter {0}: Test execution complete", AdapterVersion)); Unload(); } }
bool GetTestCaseFilterExpression(IRunContext runContext, LoggerHelper logger, string assemblyFileName, out ITestCaseFilterExpression filter) { filter = null; try { // In Microsoft.VisualStudio.TestPlatform.ObjectModel V11 IRunContext provides a TestCaseFilter property // GetTestCaseFilter only exists in V12+ #if PLATFORM_DOTNET var getTestCaseFilterMethod = runContext.GetType().GetRuntimeMethod("GetTestCaseFilter", new[] { typeof(IEnumerable<string>), typeof(Func<string, TestProperty>) }); #else var getTestCaseFilterMethod = runContext.GetType().GetMethod("GetTestCaseFilter"); #endif if (getTestCaseFilterMethod != null) filter = (ITestCaseFilterExpression)getTestCaseFilterMethod.Invoke(runContext, new object[] { supportedPropertyNames, null }); return true; } catch (TargetInvocationException e) { var innerExceptionType = e.InnerException.GetType(); if (innerExceptionType.FullName.EndsWith("TestPlatformFormatException", StringComparison.OrdinalIgnoreCase)) { logger.LogError("{0}: Exception discovering tests: {1}", Path.GetFileNameWithoutExtension(assemblyFileName), e.InnerException.Message); return false; } throw; } }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { Guard.ArgumentNotNull("sources", sources); Guard.ArgumentNotNull("runContext", runContext); Guard.ArgumentNotNull("frameworkHandle", frameworkHandle); var cleanupList = new List<ExecutorWrapper>(); try { RemotingUtility.CleanUpRegisteredChannels(); cancelled = false; foreach (string source in sources) if (VsTestRunner.IsXunitTestAssembly(source)) RunTestsInAssembly(cleanupList, source, frameworkHandle); } finally { Thread.Sleep(1000); foreach (var executorWrapper in cleanupList) executorWrapper.Dispose(); } }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { SetupExecutionPolicy(); IEnumerable<TestCase> tests = PowerShellTestDiscoverer.GetTests(sources, null); RunTests(tests, runContext, frameworkHandle); }
public TestCaseFilter(IRunContext runContext, ISet<string> traitNames, TestEnvironment testEnvironment) { _runContext = runContext; _testEnvironment = testEnvironment; InitProperties(traitNames); }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { //Debugger.Launch(); frameworkHandle.SendMessage(TestMessageLevel.Informational, Strings.EXECUTOR_STARTING); Settings settings = GetSettings(runContext); foreach (string currentAsssembly in sources.Distinct()) { try { if (!File.Exists(Path.Combine(Path.GetDirectoryName(Path.GetFullPath(currentAsssembly)),"Machine.Specifications.dll"))) { frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("Machine.Specifications.dll not found for {0}", currentAsssembly)); continue; } frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format(Strings.EXECUTOR_EXECUTINGIN, currentAsssembly)); this.executor.RunAssembly(currentAsssembly, settings, uri, frameworkHandle); } catch (Exception ex) { frameworkHandle.SendMessage(TestMessageLevel.Error, String.Format(Strings.EXECUTOR_ERROR, currentAsssembly, ex.Message)); } } frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("Complete on {0} assemblies ", sources.Count())); }
public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { //Debugger.Launch(); frameworkHandle.SendMessage(TestMessageLevel.Informational, Strings.EXECUTOR_STARTING); int executedSpecCount = 0; Settings settings = GetSettings(runContext); string currentAsssembly = string.Empty; try { foreach (IGrouping<string, TestCase> grouping in tests.GroupBy(x => x.Source)) { currentAsssembly = grouping.Key; frameworkHandle.SendMessage(TestMessageLevel.Informational, string.Format(Strings.EXECUTOR_EXECUTINGIN, currentAsssembly)); List<VisualStudioTestIdentifier> testsToRun = grouping.Select(test => test.ToVisualStudioTestIdentifier()).ToList(); this.executor.RunAssemblySpecifications(currentAsssembly, testsToRun, settings, uri, frameworkHandle); executedSpecCount += grouping.Count(); } frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format(Strings.EXECUTOR_COMPLETE, executedSpecCount, tests.GroupBy(x => x.Source).Count())); } catch (Exception ex) { frameworkHandle.SendMessage(TestMessageLevel.Error, string.Format(Strings.EXECUTOR_ERROR, currentAsssembly, ex.Message)); } finally { } }
public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { this.frameworkHandle = frameworkHandle; var testLogger = new TestLogger(frameworkHandle); testLogger.SendMainMessage("Execution started"); foreach (var group in tests.GroupBy(t => t.Source)) { testLogger.SendInformationalMessage(String.Format("Running selected: '{0}'", group.Key)); try { using (var sandbox = new Sandbox<Executor>(group.Key)) { var assemblyDirectory = new DirectoryInfo(Path.GetDirectoryName(group.Key)); Directory.SetCurrentDirectory(assemblyDirectory.FullName); sandbox.Content.Execute(this, group.Select(t => t.FullyQualifiedName).ToArray()); } } catch (Exception ex) { testLogger.SendErrorMessage(ex, String.Format("Exception found while executing tests in group '{0}'", group.Key)); // just go on with the next } } testLogger.SendMainMessage("Execution finished"); }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { this.frameworkHandle = frameworkHandle; var testLogger = new TestLogger(frameworkHandle); testLogger.SendMainMessage("Execution started"); foreach (var source in sources) { try { using (var sandbox = new Sandbox<Executor>(source)) { testLogger.SendInformationalMessage(String.Format("Running: '{0}'", source)); var assemblyDirectory = new DirectoryInfo(Path.GetDirectoryName(source)); Directory.SetCurrentDirectory(assemblyDirectory.FullName); sandbox.Content.Execute(this); } } catch (Exception ex) { testLogger.SendErrorMessage(ex, String.Format("Exception found while executing tests in source '{0}'", source)); // just go on with the next } } testLogger.SendMainMessage("Execution finished"); }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { IMessageLogger log = frameworkHandle; log.Version(); HandlePoorVisualStudioImplementationDetails(runContext, frameworkHandle); foreach (var assemblyPath in sources) { try { if (AssemblyDirectoryContainsFixie(assemblyPath)) { log.Info("Processing " + assemblyPath); var listener = new VisualStudioListener(frameworkHandle, assemblyPath); using (var environment = new ExecutionEnvironment(assemblyPath)) { environment.RunAssembly(new Options(), listener); } } else { log.Info("Skipping " + assemblyPath + " because it is not a test assembly."); } } catch (Exception exception) { log.Error(exception); } } }
/// <summary> /// Called by the Visual Studio IDE to run all tests. Also called by TFS Build /// to run either all or selected tests. In the latter case, a filter is provided /// as part of the run context. /// </summary> /// <param name="sources">Sources to be run.</param> /// <param name="runContext">Context to use when executing the tests.</param> /// <param name="frameworkHandle">Test log to send results and messages through</param> public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { testLog.Initialize(frameworkHandle); Info("executing tests", "started"); try { // Ensure any channels registered by other adapters are unregistered CleanUpRegisteredChannels(); var tfsfilter = new TFSTestFilter(runContext); testLog.SendDebugMessage("Keepalive:" + runContext.KeepAlive); if (!tfsfilter.HasTfsFilterValue && runContext.KeepAlive) frameworkHandle.EnableShutdownAfterTestRun = true; foreach (var source in sources) { using (currentRunner = new AssemblyRunner(testLog, source, tfsfilter)) { currentRunner.RunAssembly(frameworkHandle); } currentRunner = null; } } catch (Exception ex) { testLog.SendErrorMessage("Exception " + ex); } finally { Info("executing tests", "finished"); } }
/// <summary> /// Runs the tests. /// </summary> /// <param name="tests">Tests to be run.</param> /// <param name="runContext">Context to use when executing the tests.</param> /// <param param name="frameworkHandle">Handle to the framework to record results and to do framework operations.</param> public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { m_cancelled = false; try { foreach (TestCase test in tests) { if (m_cancelled) { break; } frameworkHandle.RecordStart(test); frameworkHandle.SendMessage(TestMessageLevel.Informational, "Starting external test for " + test.DisplayName); var testOutcome = RunExternalTest(test, runContext, frameworkHandle, test); frameworkHandle.RecordResult(testOutcome); frameworkHandle.SendMessage(TestMessageLevel.Informational, "Test result:" + testOutcome.Outcome.ToString()); } } catch(Exception e) { frameworkHandle.SendMessage(TestMessageLevel.Error, "Exception during test execution: " +e.Message); } }
public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { frameworkHandle.SendMessage(TestMessageLevel.Informational, Strings.EXECUTOR_STARTING); int executedSpecCount = 0; string currentAsssembly = string.Empty; try { ISpecificationExecutor specificationExecutor = this.adapterFactory.CreateExecutor(); IEnumerable<IGrouping<string, TestCase>> groupBySource = tests.GroupBy(x => x.Source); foreach (IGrouping<string, TestCase> grouping in groupBySource) { currentAsssembly = grouping.Key; frameworkHandle.SendMessage(TestMessageLevel.Informational, string.Format(Strings.EXECUTOR_EXECUTINGIN, currentAsssembly)); specificationExecutor.RunAssemblySpecifications(currentAsssembly, MSpecTestAdapter.uri, runContext, frameworkHandle, grouping); executedSpecCount += grouping.Count(); } frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format(Strings.EXECUTOR_COMPLETE, executedSpecCount, groupBySource.Count())); } catch (Exception ex) { frameworkHandle.SendMessage(TestMessageLevel.Error, string.Format(Strings.EXECUTOR_ERROR, currentAsssembly, ex.Message)); } finally { } }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { ChutzpahTracer.TraceInformation("Begin Test Adapter Run Tests"); var settingsProvider = runContext.RunSettings.GetSettings(AdapterConstants.SettingsName) as ChutzpahAdapterSettingsProvider; var settings = settingsProvider != null ? settingsProvider.Settings : new ChutzpahAdapterSettings(); ChutzpahTracingHelper.Toggle(settings.EnabledTracing); var testOptions = new TestOptions { TestLaunchMode = runContext.IsBeingDebugged ? TestLaunchMode.Custom: settings.OpenInBrowser ? TestLaunchMode.FullBrowser: TestLaunchMode.HeadlessBrowser, CustomTestLauncher = runContext.IsBeingDebugged ? new VsDebuggerTestLauncher() : null, MaxDegreeOfParallelism = runContext.IsBeingDebugged ? 1 : settings.MaxDegreeOfParallelism, ChutzpahSettingsFileEnvironments = new ChutzpahSettingsFileEnvironments(settings.ChutzpahSettingsFileEnvironments) }; testOptions.CoverageOptions.Enabled = runContext.IsDataCollectionEnabled; var callback = new ParallelRunnerCallbackAdapter(new ExecutionCallback(frameworkHandle, runContext)); testRunner.RunTests(sources, testOptions, callback); ChutzpahTracer.TraceInformation("End Test Adapter Run Tests"); }
public void Cleanup() { _runContext = null; foreach (var adapter in GetAdapters()) { adapter.Cleanup(); } _adapters.Clear(); }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { List<TestCase> tests = new List<TestCase>(); TestDiscoverer.VisitTests(sources, t => tests.Add(t)); InternalRunTests(tests, runContext, frameworkHandle, null); }
public ServersTree(IRunContext runContext) { Context = runContext; Children = new ObservableCollection<Node> { new LocalHostNode(this), }; }
public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { ValidateArg.NotNull(tests, "tests"); ValidateArg.NotNull(runContext, "runContext"); ValidateArg.NotNull(frameworkHandle, "frameworkHandle"); _cancelRequested.Reset(); RunTestCases(tests, runContext, frameworkHandle); }
private void RunTests(string source, IRunContext runContext, IFrameworkHandle frameworkHandle) { foreach (var result in ExternalTestExecutor.GetTestResults(source, null).Select(c => CreateTestResult(source, c))) { frameworkHandle.RecordStart(result.TestCase); frameworkHandle.RecordResult(result); frameworkHandle.RecordEnd(result.TestCase, result.Outcome); } }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { RunTests(sources.Select(x => new TestCase(x, ExecutorDetails.Uri, x) { CodeFilePath = x, LineNumber = 2, DisplayName = "Abra cadabra" }), runContext, frameworkHandle); }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, ITestExecutionRecorder testExecutionRecorder) { Guard.ArgumentNotNull("sources", sources); Guard.ArgumentNotNull("runContext", runContext); Guard.ArgumentNotNull("testExecutionRecorder", testExecutionRecorder); foreach (string source in sources) if (VsTestRunner.IsXunitTestAssembly(source)) RunTestsInAssembly(source, runContext, testExecutionRecorder); }
public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, ITestExecutionRecorder testExecutionRecorder) { Guard.ArgumentNotNull("tests", tests); Guard.ArgumentNotNull("runContext", runContext); Guard.ArgumentNotNull("testExecutionRecorder", testExecutionRecorder); foreach (var testCaseGroup in tests.GroupBy(tc => tc.Source)) if (VsTestRunner.IsXunitTestAssembly(testCaseGroup.Key)) RunTestsInAssembly(testCaseGroup.Key, runContext, testExecutionRecorder, testCaseGroup); }
private void RunTests(IRunContext runContext, IFrameworkHandle testExecutionRecorder) { if (runContext.InIsolation) launcher.TestProject.TestRunnerFactoryName = StandardTestRunnerFactoryNames.IsolatedAppDomain; var extension = new VSTestWindowExtension(testExecutionRecorder, testCaseFactory, testResultFactory); launcher.TestProject.AddTestRunnerExtension(extension); launcher.Run(); }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { //Debugger.Launch(); // this a temporary hack until I can figure out why running the specs per assembly directly using mspec does not work with a large number of specifications // when they are run diectly the first 100 or so specs run fine and then an error occurs saying it has taken more than 10 seconds and is being stopped // for now we just rediscover and run them like that, makes no sense TestCaseCollector collector = new TestCaseCollector(); this.DiscoverTests(sources, runContext, frameworkHandle, collector); this.RunTests(collector.TestCases, runContext, frameworkHandle); }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { var validSources = from source in sources where source.EndsWith(StringHelper.GetSearchExpression(), StringComparison.CurrentCultureIgnoreCase) select source; foreach (var source in validSources) { RunTest(frameworkHandle, source); } }
public TestGeneratorAdapterTests() { testGeneratorDiscoverer = Substitute.For<ITestGeneratorDiscoverer>(); discoveryContext = Substitute.For<IDiscoveryContext>(); discoverySink = Substitute.For<ITestCaseDiscoverySink>(); logger = Substitute.For<IMessageLogger>(); frameworkHandle = Substitute.For<IFrameworkHandle>(); runContext = Substitute.For<IRunContext>(); testGenerator = Substitute.For<ITestGenerator>(); }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle testExecutionRecorder) { launcher = new TestLauncher(); foreach (var source in sources) { launcher.AddFilePattern(source); } RunTests(runContext, testExecutionRecorder); }
private static IEnumerable<KarmaTestResult> GetTestResults(IRunContext runContext) { var firstResultFile = Directory.EnumerateFiles(Path.Combine(@"C:\temp", "test-results"), "*.xml").First(); string outputResults = File.ReadAllText(firstResultFile); var karmaResultsParser = new JUnitResultsParser(); IEnumerable<KarmaTestResult> testResult = karmaResultsParser.Parse(outputResults); return testResult; }
void ITestAdapter.Initialize(IRunContext runContext) { // We delay inner TAs initialization until Run method because we don't know which test type this is going to be. m_runContext = runContext; TestRunConfiguration runConfig = m_runContext.RunConfig.TestRun.RunConfiguration; Debug.Assert(runConfig != null); VsIdeHostRunConfigData runConfigHostData = runConfig.HostData[VsIdeHostAdapter.Name] as VsIdeHostRunConfigData; if (runConfigHostData != null) { VsIdeTestHostContext.AdditionalTestData = runConfigHostData.AdditionalTestData; } }
private static XDocument RunOrDebugCatchTest(string testBinary, string testSpec, IRunContext runContext, IFrameworkHandle framework) { if (runContext.IsBeingDebugged) { return(XDocument.Parse(DebugCatchTest(testBinary, testSpec, framework))); } else { return(XDocument.Parse(RunCatchTests(testBinary, testSpec))); } }
public void RunTests(IEnumerable <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { // We'll just punt and run everything in each file that contains the selected tests var sources = tests.Select(test => test.Source).Distinct(); RunTests(sources, runContext, frameworkHandle); }
public void Initialize(IRunContext runContext) { _runContext = runContext; _runId = runContext.RunConfig.TestRun.Id; }
public void RunTests(IEnumerable <string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { RunTestsWithSourcesCallback?.Invoke(sources, runContext, frameworkHandle); }
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { // HACK: For now, just run them all - we can fix this later :) RunTests(tests.Select(t => t.Source).Distinct(), runContext, frameworkHandle); }
void RunTests(IRunContext runContext, IFrameworkHandle frameworkHandle, Stopwatch stopwatch, Func <XunitVisualStudioSettings, IEnumerable <IGrouping <string, TestCase> > > testCaseAccessor) { Guard.ArgumentNotNull("runContext", runContext); Guard.ArgumentNotNull("frameworkHandle", frameworkHandle); var settings = SettingsProvider.Load(); var shuttingDown = !runContext.KeepAlive || settings.ShutdownAfterRun; if (runContext.KeepAlive && settings.ShutdownAfterRun) { frameworkHandle.EnableShutdownAfterTestRun = true; } var toDispose = new List <IDisposable>(); if (settings.MessageDisplay == MessageDisplay.Diagnostic) { lock (stopwatch) { frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Execution started", stopwatch.Elapsed)); frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Settings: MaxParallelThreads = {1}, NameDisplay = {2}, ParallelizeAssemblies = {3}, ParallelizeTestCollections = {4}, ShutdownAfterRun = {5}", stopwatch.Elapsed, settings.MaxParallelThreads, settings.NameDisplay, settings.ParallelizeAssemblies, settings.ParallelizeTestCollections, settings.ShutdownAfterRun)); } } try { RemotingUtility.CleanUpRegisteredChannels(); cancelled = false; using (AssemblyHelper.SubscribeResolve()) if (settings.ParallelizeAssemblies) { testCaseAccessor(settings) .Select(testCaseGroup => RunTestsInAssemblyAsync(runContext, frameworkHandle, toDispose, testCaseGroup.Key, testCaseGroup, settings, stopwatch)) .ToList() .ForEach(@event => @event.WaitOne()); } else { testCaseAccessor(settings) .ToList() .ForEach(testCaseGroup => RunTestsInAssembly(runContext, frameworkHandle, toDispose, testCaseGroup.Key, testCaseGroup, settings, stopwatch)); } } finally { if (!shuttingDown) { toDispose.ForEach(disposable => disposable.Dispose()); } } if (settings.MessageDisplay == MessageDisplay.Diagnostic) { lock (stopwatch) frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Execution complete", stopwatch.Elapsed)); } }
public bool ShouldAttachToTestHost(IEnumerable <VsTestCase> tests, IRunContext runContext) { // TODO: expose setting in runContext to attach to testhost if needed? return(false); }
private void RunTestCase( IFrameworkHandle frameworkHandle, IRunContext runContext, TestCase test, Dictionary <string, PythonProjectSettings> sourceToSettings ) { var testResult = new TestResult(test); frameworkHandle.RecordStart(test); testResult.StartTime = DateTimeOffset.Now; PythonProjectSettings settings; if (!sourceToSettings.TryGetValue(test.Source, out settings)) { sourceToSettings[test.Source] = settings = LoadProjectSettings(test.Source, _interpreterService); } if (settings == null) { frameworkHandle.SendMessage( TestMessageLevel.Error, "Unable to determine interpreter to use for " + test.Source); RecordEnd( frameworkHandle, test, testResult, null, "Unable to determine interpreter to use for " + test.Source, TestOutcome.Failed); return; } var debugMode = PythonDebugMode.None; if (runContext.IsBeingDebugged && _app != null) { debugMode = settings.EnableNativeCodeDebugging ? PythonDebugMode.PythonAndNative : PythonDebugMode.PythonOnly; } var testCase = new PythonTestCase(settings, test, debugMode); var dte = _app.GetDTE(); if (debugMode != PythonDebugMode.None) { dte.Debugger.DetachAll(); } if (!File.Exists(settings.Factory.Configuration.InterpreterPath)) { frameworkHandle.SendMessage(TestMessageLevel.Error, "Interpreter path does not exist: " + settings.Factory.Configuration.InterpreterPath); return; } var env = new Dictionary <string, string>(); var pythonPathVar = settings.Factory.Configuration.PathEnvironmentVariable; var pythonPath = testCase.SearchPaths; if (!string.IsNullOrWhiteSpace(pythonPathVar)) { if (_app != null) { var settingsManager = SettingsManagerCreator.GetSettingsManager(dte); if (settingsManager != null) { var store = settingsManager.GetReadOnlySettingsStore(SettingsScope.UserSettings); if (store != null && store.CollectionExists(@"PythonTools\Options\General")) { var settingStr = store.GetString(@"PythonTools\Options\General", "ClearGlobalPythonPath", "True"); bool settingBool; if (bool.TryParse(settingStr, out settingBool) && !settingBool) { pythonPath += ";" + Environment.GetEnvironmentVariable(pythonPathVar); } } } } env[pythonPathVar] = pythonPath; } foreach (var envVar in testCase.Environment) { env[envVar.Key] = envVar.Value; } using (var proc = ProcessOutput.Run( !settings.IsWindowsApplication ? settings.Factory.Configuration.InterpreterPath : settings.Factory.Configuration.WindowsInterpreterPath, testCase.Arguments, testCase.WorkingDirectory, env, false, null )) { bool killed = false; #if DEBUG frameworkHandle.SendMessage(TestMessageLevel.Informational, "cd " + testCase.WorkingDirectory); frameworkHandle.SendMessage(TestMessageLevel.Informational, "set " + (pythonPathVar ?? "") + "=" + (pythonPath ?? "")); frameworkHandle.SendMessage(TestMessageLevel.Informational, proc.Arguments); #endif proc.Wait(TimeSpan.FromMilliseconds(500)); if (debugMode != PythonDebugMode.None) { if (proc.ExitCode.HasValue) { // Process has already exited frameworkHandle.SendMessage(TestMessageLevel.Error, "Failed to attach debugger because the process has already exited."); if (proc.StandardErrorLines.Any()) { frameworkHandle.SendMessage(TestMessageLevel.Error, "Standard error from Python:"); foreach (var line in proc.StandardErrorLines) { frameworkHandle.SendMessage(TestMessageLevel.Error, line); } } } try { if (debugMode == PythonDebugMode.PythonOnly) { string qualifierUri = string.Format("tcp://{0}@localhost:{1}", testCase.DebugSecret, testCase.DebugPort); while (!_app.AttachToProcess(proc, PythonRemoteDebugPortSupplierUnsecuredId, qualifierUri)) { if (proc.Wait(TimeSpan.FromMilliseconds(500))) { break; } } } else { var engines = new[] { PythonDebugEngineGuid, VSConstants.DebugEnginesGuids.NativeOnly_guid }; while (!_app.AttachToProcess(proc, engines)) { if (proc.Wait(TimeSpan.FromMilliseconds(500))) { break; } } } #if DEBUG } catch (COMException ex) { frameworkHandle.SendMessage(TestMessageLevel.Error, "Error occurred connecting to debuggee."); frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString()); try { proc.Kill(); } catch (InvalidOperationException) { // Process has already exited } killed = true; } #else } catch (COMException) {
void RunTestsInAssembly(IRunContext runContext, IFrameworkHandle frameworkHandle, LoggerHelper logger, TestPlatformContext testPlatformContext, RunSettings runSettings, IMessageSinkWithTypes reporterMessageHandler, AssemblyRunInfo runInfo) { if (cancelled) { return; } var assemblyDisplayName = "(unknown assembly)"; try { var assembly = new XunitProjectAssembly { AssemblyFilename = runInfo.AssemblyFileName }; var assemblyFileName = runInfo.AssemblyFileName; assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName); var configuration = runInfo.Configuration; var shadowCopy = configuration.ShadowCopyOrDefault; var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; if (runSettings.DisableAppDomain) { appDomain = AppDomainSupport.Denied; } #if WINDOWS_UAP // For AppX Apps, use the package location assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName)); #endif var diagnosticSink = DiagnosticMessageSink.ForDiagnostics(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault); var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink); using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) { var testCasesMap = new Dictionary <string, TestCase>(); var testCases = new List <ITestCase>(); if (runInfo.TestCases == null || !runInfo.TestCases.Any()) { // Discover tests AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo(); DiscoverTestsInSource(controller, logger, testPlatformContext, runSettings, (source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled), (source, discoverer, discoveryOptions, visitor) => { if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault()) { foreach (var testCase in visitor.TestCases) { logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')", testCase.DisplayName, testCase.UniqueID); } } assemblyDiscoveredInfo = new AssemblyDiscoveredInfo { AssemblyFileName = source, DiscoveredTestCases = GetVsTestCases(source, discoverer, visitor, logger, testPlatformContext) }; }, assemblyFileName, shadowCopy, configuration ); if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any()) { if (configuration.InternalDiagnosticMessagesOrDefault) { logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName); } return; } // Filter tests var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames)); var filter = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames); var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList(); foreach (var filteredTestCase in filteredTestCases) { var uniqueID = filteredTestCase.UniqueID; if (testCasesMap.ContainsKey(uniqueID)) { logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName); } else { testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase); testCases.Add(filteredTestCase.TestCase); } } } else { try { var serializations = runInfo.TestCases .Select(tc => tc.GetPropertyValue <string>(SerializedTestCaseProperty, null)) .ToList(); if (configuration.InternalDiagnosticMessagesOrDefault) { logger.LogWithSource(runInfo.AssemblyFileName, "Deserializing {0} test case(s):{1}{2}", serializations.Count, Environment.NewLine, string.Join(Environment.NewLine, serializations.Select(x => $" {x}"))); } var deserializedTestCasesByUniqueId = controller.BulkDeserialize(serializations); if (deserializedTestCasesByUniqueId == null) { logger.LogErrorWithSource(assemblyFileName, "Received null response from BulkDeserialize"); } else { for (int idx = 0; idx < runInfo.TestCases.Count; ++idx) { try { var kvp = deserializedTestCasesByUniqueId[idx]; var vsTestCase = runInfo.TestCases[idx]; if (kvp.Value == null) { logger.LogErrorWithSource(assemblyFileName, "Test case {0} failed to deserialize: {1}", vsTestCase.DisplayName, kvp.Key); } else { testCasesMap.Add(kvp.Key, vsTestCase); testCases.Add(kvp.Value); } } catch (Exception ex) { logger.LogErrorWithSource(assemblyFileName, "Catastrophic error deserializing item #{0}: {1}", idx, ex); } } } } catch (Exception ex) { logger.LogWarningWithSource(assemblyFileName, "Catastrophic error during deserialization: {0}", ex); } } // Execute tests var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration); if (runSettings.DisableParallelization) { executionOptions.SetSynchronousMessageReporting(true); executionOptions.SetDisableParallelization(true); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled)) { IExecutionSink resultsSink = vsExecutionSink; if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink); } controller.RunTests(testCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); } } } catch (Exception ex) { logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex); } }
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { Debugger.Launch(); }
public void RunTests(IEnumerable <string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { Debugger.Launch(); }
/// <summary> /// Execute the parameter tests present in parameter source /// </summary> /// <param name="tests">Tests to execute.</param> /// <param name="runContext">The run context.</param> /// <param name="frameworkHandle">Handle to record test start/end/results.</param> /// <param name="source">The test container for the tests.</param> /// <param name="isDeploymentDone">Indicates if deployment is done.</param> private void ExecuteTestsInSource(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle, string source, bool isDeploymentDone) { Debug.Assert(!string.IsNullOrEmpty(source), "Source cannot be empty"); if (isDeploymentDone) { source = Path.Combine(PlatformServiceProvider.Instance.TestDeployment.GetDeploymentDirectory(), Path.GetFileName(source)); } using (var isolationHost = PlatformServiceProvider.Instance.CreateTestSourceHost(source, runContext?.RunSettings, frameworkHandle)) { var testRunner = isolationHost.CreateInstanceForType( typeof(UnitTestRunner), new object[] { MSTestSettings.CurrentSettings }) as UnitTestRunner; PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Created unit-test runner {0}", source); // Default test set is filtered tests based on user provided filter criteria IEnumerable <TestCase> testsToRun = Enumerable.Empty <TestCase>(); var filterExpression = this.TestMethodFilter.GetFilterExpression(runContext, frameworkHandle, out var filterHasError); if (filterHasError) { // Bail out without processing everything else below. return; } testsToRun = tests.Where(t => MatchTestFilter(filterExpression, t, this.TestMethodFilter)); // this is done so that appropriate values of testcontext properties are set at source level // and are merged with session level parameters var sourceLevelParameters = PlatformServiceProvider.Instance.SettingsProvider.GetProperties(source); if (this.sessionParameters != null && this.sessionParameters.Count > 0) { sourceLevelParameters = sourceLevelParameters.Concat(this.sessionParameters).ToDictionary(x => x.Key, x => x.Value); } var sourceSettingsProvider = isolationHost.CreateInstanceForType( typeof(TestAssemblySettingsProvider), null) as TestAssemblySettingsProvider; var sourceSettings = sourceSettingsProvider.GetSettings(source); var parallelWorkers = sourceSettings.Workers; var parallelScope = sourceSettings.Scope; if (MSTestSettings.CurrentSettings.ParallelizationWorkers.HasValue) { // The runsettings value takes precedence over an assembly level setting. Reset the level. parallelWorkers = MSTestSettings.CurrentSettings.ParallelizationWorkers.Value; } if (MSTestSettings.CurrentSettings.ParallelizationScope.HasValue) { // The runsettings value takes precedence over an assembly level setting. Reset the level. parallelScope = MSTestSettings.CurrentSettings.ParallelizationScope.Value; } if (!MSTestSettings.CurrentSettings.DisableParallelization && sourceSettings.CanParallelizeAssembly && parallelWorkers > 0) { // Parallelization is enabled. Let's do further classification for sets. var logger = (IMessageLogger)frameworkHandle; logger.SendMessage( TestMessageLevel.Informational, string.Format(CultureInfo.CurrentCulture, Resource.TestParallelizationBanner, source, parallelWorkers, parallelScope)); // Create test sets for execution, we can execute them in parallel based on parallel settings IEnumerable <IGrouping <bool, TestCase> > testsets = Enumerable.Empty <IGrouping <bool, TestCase> >(); // Parallel and not parallel sets. testsets = testsToRun.GroupBy(t => t.GetPropertyValue <bool>(TestAdapter.Constants.DoNotParallelizeProperty, false)); var parallelizableTestSet = testsets.FirstOrDefault(g => g.Key == false); var nonparallelizableTestSet = testsets.FirstOrDefault(g => g.Key == true); if (parallelizableTestSet != null) { ConcurrentQueue <IEnumerable <TestCase> > queue = null; // Chunk the sets into further groups based on parallel level switch (parallelScope) { case ExecutionScope.MethodLevel: queue = new ConcurrentQueue <IEnumerable <TestCase> >(parallelizableTestSet.Select(t => new[] { t })); break; case ExecutionScope.ClassLevel: queue = new ConcurrentQueue <IEnumerable <TestCase> >(parallelizableTestSet.GroupBy(t => t.GetPropertyValue(TestAdapter.Constants.TestClassNameProperty) as string)); break; } var tasks = new List <Task>(); for (int i = 0; i < parallelWorkers; i++) { tasks.Add(Task.Factory.StartNew( () => { while (!queue.IsEmpty) { if (this.cancellationToken != null && this.cancellationToken.Canceled) { // if a cancellation has been requested, do not queue any more test runs. break; } if (queue.TryDequeue(out IEnumerable <TestCase> testSet)) { this.ExecuteTestsWithTestRunner(testSet, runContext, frameworkHandle, source, sourceLevelParameters, testRunner); } } }, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default)); } Task.WaitAll(tasks.ToArray()); } // Queue the non parallel set if (nonparallelizableTestSet != null) { this.ExecuteTestsWithTestRunner(nonparallelizableTestSet, runContext, frameworkHandle, source, sourceLevelParameters, testRunner); } } else { this.ExecuteTestsWithTestRunner(testsToRun, runContext, frameworkHandle, source, sourceLevelParameters, testRunner); } this.RunCleanup(frameworkHandle, testRunner); PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo( "Executed tests belonging to source {0}", source); } }
List <AssemblyRunInfo> GetTests(IEnumerable <string> sources, LoggerHelper logger, IRunContext runContext) { // For store apps, the files are copied to the AppX dir, we need to load it from there #if PLATFORM_DOTNET sources = sources.Select(s => Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(s))); #endif var result = new List <AssemblyRunInfo>(); var knownTraitNames = new HashSet <string>(StringComparer.OrdinalIgnoreCase); DiscoverTests( sources, logger, (source, discoverer, discoveryOptions) => new VsExecutionDiscoveryVisitor(), (source, discoverer, discoveryOptions, visitor) => { var vsFilteredTestCases = visitor.TestCases.Select(testCase => VsDiscoveryVisitor.CreateVsTestCase(source, discoverer, testCase, false, logger: logger, knownTraitNames: knownTraitNames)).ToList(); // Apply any filtering var filterHelper = new TestCaseFilterHelper(knownTraitNames); vsFilteredTestCases = filterHelper.GetFilteredTestList(vsFilteredTestCases, runContext, logger, source).ToList(); // Re-create testcases with unique names if there is more than 1 var testCases = visitor.TestCases.Where(tc => vsFilteredTestCases.Any(vsTc => vsTc.DisplayName == tc.DisplayName)).GroupBy(tc => $"{tc.TestMethod.TestClass.Class.Name}.{tc.TestMethod.Method.Name}") .SelectMany(group => group.Select(testCase => VsDiscoveryVisitor.CreateVsTestCase( source, discoverer, testCase, forceUniqueNames: group.Count() > 1, logger: logger, knownTraitNames: knownTraitNames)) .Where(vsTestCase => vsTestCase != null)).ToList(); // pre-enumerate these as it populates the known trait names collection var runInfo = new AssemblyRunInfo { AssemblyFileName = source, Configuration = LoadConfiguration(source), TestCases = testCases }; result.Add(runInfo); } ); return(result); }
/// <summary> /// Gets filter expression from run context. /// </summary> /// <param name="context">Run context</param> /// <returns>Filter expression.</returns> private ITestCaseFilterExpression GetTestCaseFilterFromRunContext(IRunContext context) { return(context.GetTestCaseFilter(this.supportedProperties.Keys, this.PropertyProvider)); }
/// <summary> /// Implements initialization. This is called by ITestAdapter.Initialize /// (a synchronous function) which will block until this function is /// completed. /// </summary> /// <param name="runContext"> /// The context for the current test run. /// </param> private async Task InitializeWorker( TestProperties vars, IRunContext runContext, ITestElement currentTest = null ) { string application, executable, versionString, hive; Version version; string launchTimeoutInSecondsString; int launchTimeoutInSeconds; string reuseInstanceString; // VSApplication is the registry key name like 'VisualStudio' application = vars[VSTestProperties.VSApplication.Key] ?? VSTestProperties.VSApplication.VisualStudio; // VSExecutableName is the executable name like 'devenv' if (!vars.TryGetValue(VSTestProperties.VSExecutable.Key, out executable)) { executable = VSTestProperties.VSExecutable.DevEnv; } if (!string.IsNullOrEmpty(executable) && string.IsNullOrEmpty(Path.GetExtension(executable))) { executable = Path.ChangeExtension(executable, ".exe"); } // VSVersion is the version like '12.0' if (!vars.TryGetValue(VSTestProperties.VSVersion.Key, out versionString) || !Version.TryParse(versionString, out version)) { version = new Version(int.Parse(AssemblyVersionInfo.VSVersion), 0); } // VSHive is the optional hive like 'Exp' hive = vars[VSTestProperties.VSHive.Key]; if (!vars.TryGetValue(VSTestProperties.VSLaunchTimeoutInSeconds.Key, out launchTimeoutInSecondsString) || !int.TryParse(launchTimeoutInSecondsString, out launchTimeoutInSeconds)) { launchTimeoutInSeconds = 30; } if (string.IsNullOrEmpty(application) || string.IsNullOrEmpty(executable) || version == null) { throw new ArgumentException(string.Format( Resources.MissingConfigurationValues, application ?? "(null)", executable ?? "(null)", version != null ? version.ToString() : "(null)", hive ?? "(default)" )); } _mockVs = (application == VSTestProperties.VSApplication.Mock); if (_mockVs) { _remote = new TesteeTestAdapter(); _remote.Initialize(_runContext); // In the mock case tester and testee are the same process, therefore // VSTestContext is in our process too. So we can just set this value // directly here. VSTestContext.IsMock = true; return; } if (!vars.TryGetValue(VSTestProperties.VSReuseInstance.Key, out reuseInstanceString) || !bool.TryParse(reuseInstanceString, out _reuseInstance)) { // the default behavior is to reuse the current instance when possible _reuseInstance = true; } // TODO: Detect and perform first run of VS if necessary. // The first time a VS hive is run, the user sees a dialog allowing // them to select settings such as the theme. We can avoid this by // running devenv.exe /resetSettings <path to profile.settings> // first, though it is not trivial to detect when this is necessary. // Without having done this, all tests will time out. For now, the // user is responsible for running VS at least once before // attempting to execute tests. var cts = new CancellationTokenSource(TimeSpan.FromSeconds(launchTimeoutInSeconds)); try { await Connect(application, executable, version, hive, runContext, currentTest, cts.Token); } catch (OperationCanceledException ex) { throw new TimeoutException(string.Format(Resources.VSLaunchTimeout, launchTimeoutInSeconds), ex); } catch (Exception ex) { throw new InvalidOperationException( string.Format(Resources.VSFailedToLaunch, application, executable, version, hive), ex ); } }
public void RunTests(IEnumerable <string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { }
public FeatureRunner(IStringStepRunner stringStepRunner, IRunContext context) { this.context = context; this.stringStepRunner = stringStepRunner; }
private void RunTestsCore(IEnumerable <ITestScript> tests, Predicate <ITestFunction> filter, ILogger logger, IRunContext runContext, ITestExecutionRecorder recorder) { var settingsService = runContext.RunSettings.GetSettings(PSycheTestRunSettings.SettingsProviderName) as IPSycheTestSettingsService ?? new PSycheTestSettingsService(); var testCaseMap = new Dictionary <ITestFunction, TestCase>(); var executor = _executorFactory(logger); executor.OutputDirectory = new DirectoryInfo(runContext.TestRunDirectory); logger.Info("Test output directory: {0}", executor.OutputDirectory.FullName); foreach (var module in settingsService.Settings.Modules) { logger.Info("Adding module '{0}'", module); executor.InitialModules.Add(module); } executor.TestStarting += (o, e) => { var testCase = _mapper.Map(e.Test); testCaseMap[e.Test] = testCase; recorder.RecordStart(testCase); }; executor.TestEnded += (o, e) => { var testCase = testCaseMap[e.Test]; recorder.RecordEnd(testCase, _mapper.Map(e.Result.Status)); recorder.RecordResult(_mapper.Map(testCase, e.Result)); }; using (_cancellationTokenSource = new CancellationTokenSource()) { executor.ExecuteAsync(tests, filter, _cancellationTokenSource.Token).Wait(); // Awaiting seems to cause odd behavior, just block instead. } _cancellationTokenSource = null; }
void RunTestsInAssembly(IRunContext runContext, IFrameworkHandle frameworkHandle, LoggerHelper logger, TestPlatformContext testPlatformContext, IMessageSinkWithTypes reporterMessageHandler, AssemblyRunInfo runInfo) { if (cancelled) { return; } var assembly = new XunitProjectAssembly { AssemblyFilename = runInfo.AssemblyFileName }; var assemblyFileName = runInfo.AssemblyFileName; var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName); var configuration = runInfo.Configuration; var shadowCopy = configuration.ShadowCopyOrDefault; var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; if (RunSettingsHelper.DisableAppDomain) { appDomain = AppDomainSupport.Denied; } try { #if WINDOWS_UAP // For AppX Apps, use the package location assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName)); #endif var diagnosticSink = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault); var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink); using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) { var testCasesMap = new Dictionary <string, TestCase>(); var testCases = new List <ITestCase>(); if (runInfo.TestCases == null || !runInfo.TestCases.Any()) { // Discover tests AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo(); DiscoverTestsInSource(controller, logger, testPlatformContext, (source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled), (source, discoverer, discoveryOptions, visitor) => { if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault()) { foreach (var testCase in visitor.TestCases) { logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')", testCase.DisplayName, testCase.UniqueID); } } assemblyDiscoveredInfo = new AssemblyDiscoveredInfo { AssemblyFileName = source, DiscoveredTestCases = visitor.TestCases.Select(testCase => new DiscoveredTestCase(source, discoverer, testCase, logger, testPlatformContext)).ToList() }; }, assemblyFileName, shadowCopy, configuration ); if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any()) { if (configuration.InternalDiagnosticMessagesOrDefault) { logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName); } return; } // Filter tests var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames)); // Apply any filtering var filter = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames); var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList(); // Force unique names if there is more than 1 testcase with the same name foreach (var groupWithDuplicateNames in filteredTestCases.GroupBy(dtc => dtc.Name).Where(group => group.Count() > 1)) { foreach (var discoveredTestCaseWithDuplicateName in groupWithDuplicateNames) { discoveredTestCaseWithDuplicateName.ForceUniqueName(); } } foreach (var filteredTestCase in filteredTestCases) { var uniqueID = filteredTestCase.UniqueID; if (testCasesMap.ContainsKey(uniqueID)) { logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName); } else { testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase); testCases.Add(filteredTestCase.TestCase); } } } else { // We are in Run Specific tests scenario, the `TestCase` objects are available. // Query the `TestCase` objects to find XunitTestCase objects. foreach (var vstestCase in runInfo.TestCases) { var xunitTestCase = Deserialize(logger, controller, vstestCase); if (xunitTestCase != null) { testCasesMap.Add(xunitTestCase.UniqueID, vstestCase); testCases.Add(xunitTestCase); } } } // Execute tests var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration); if (RunSettingsHelper.DisableParallelization) { executionOptions.SetSynchronousMessageReporting(true); executionOptions.SetDisableParallelization(true); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled)) { IExecutionSink resultsSink = vsExecutionSink; if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink); } controller.RunTests(testCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); } } } catch (Exception ex) { logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex); } }
private FeatureResults Run(IFeatureRunner featureRunner, IEnumerable <Feature> features, IRunContext context) { var result = new FeatureResults(); foreach (Feature feature in features) { FeatureResult featureResult = null; try { context.FeatureStarted(feature); featureResult = featureRunner.Run(feature); result.Add(featureResult); } finally { context.FeatureFinished(featureResult); } } return(result); }
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { RunTestsWithTestsCallback?.Invoke(tests, runContext, frameworkHandle); }
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { _mCancelled = false; SetupExecutionPolicy(); var testSets = new List <TestCaseSet>(); foreach (var testCase in tests) { var describe = testCase.FullyQualifiedName.Split('.').First(); var codeFile = testCase.CodeFilePath; var testSet = testSets.FirstOrDefault(m => m.Describe.Equals(describe, StringComparison.OrdinalIgnoreCase) && m.File.Equals(codeFile, StringComparison.OrdinalIgnoreCase)); if (testSet == null) { testSet = new TestCaseSet(codeFile, describe); testSets.Add(testSet); } testSet.TestCases.Add(testCase); } foreach (var testSet in testSets) { if (_mCancelled) { break; } var testOutput = new StringBuilder(); try { var testAdapter = new TestAdapterHost(); testAdapter.HostUi.OutputString = s => { if (!string.IsNullOrEmpty(s)) { testOutput.Append(s); } }; var runpsace = RunspaceFactory.CreateRunspace(testAdapter); runpsace.Open(); using (var ps = PowerShell.Create()) { ps.Runspace = runpsace; RunTestSet(ps, testSet, runContext); foreach (var testResult in testSet.TestResults) { frameworkHandle.RecordResult(testResult); } } } catch (Exception ex) { foreach (var testCase in testSet.TestCases) { var testResult = new TestResult(testCase); testResult.Outcome = TestOutcome.Failed; testResult.ErrorMessage = ex.Message; testResult.ErrorStackTrace = ex.StackTrace; frameworkHandle.RecordResult(testResult); } } if (testOutput.Length > 0) { frameworkHandle.SendMessage(TestMessageLevel.Informational, testOutput.ToString()); } } }
/// <summary> /// ITestAdapter method: called just before the test run finishes and /// gives the adapter a chance to do any clean-up. /// </summary> /// <param name="runContext">The run context for this run</param> void ITestAdapter.PreTestRunFinished(IRunContext runContext) { HostSide.PreTestRunFinished(runContext); }
public void PreTestRunFinished(IRunContext runContext) { RemoteCall(r => r.PreTestRunFinished(runContext)); }
public void RunTests(IEnumerable <string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { throw new NotImplementedException(); }
private async Task Connect( string application, string executable, Version version, string hive, IRunContext runContext, ITestElement currentTest, CancellationToken cancel ) { var hiveOption = string.IsNullOrEmpty(hive) ? "" : (" /rootSuffix " + hive); if (_reuseInstance && _ide != null && _remote != null && application == _currentApplication && executable == _currentExecutable && version == _currentVersion && hive == _currentHive) { if (runContext != null) { SendMessage( runContext, string.Format(Resources.VSReuseMessage, application, executable, version, hiveOption), currentTest ); } return; } Close(); if (runContext != null) { SendMessage( runContext, string.Format(Resources.VSLaunchMessage, application, executable, version, hiveOption), currentTest ); } Internal.VisualStudio ide = null; try { ide = await Internal.VisualStudio.LaunchAsync(application, executable, version, hive, cancel); var p = Process.GetProcessById(ide.ProcessId); var url = string.Format("ipc://{0}/{1}", VSTestHostPackage.GetChannelName(p), TesteeTestAdapter.Url); try { _remote = (TesteeTestAdapter)RemotingServices.Connect(typeof(TesteeTestAdapter), url); } catch (RemotingException ex) { throw new InvalidOperationException(Resources.FailedToConnect, ex); } _currentApplication = application; _currentExecutable = executable; _currentVersion = version; _currentHive = hive; _ide = ide; ide = null; } finally { if (ide != null) { ide.Dispose(); } } }
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { throw new NotImplementedException(); }
private string RunProtractor(TestCase test, IRunContext runContext, IFrameworkHandle frameworkHandle) { var resultFile = Path.GetFileNameWithoutExtension(test.CodeFilePath); resultFile += ".result.json"; resultFile = AppConfig.ResultsPath ?? Path.GetTempPath() + Path.DirectorySeparatorChar + resultFile; frameworkHandle.SendMessage(TestMessageLevel.Informational, "Framework: Using result file: " + resultFile); // Can't use test.Source as it's going to be in lowercase var cwd = Helper.FindInDirectoryTree(test.CodeFilePath, "package.json"); var exe = Helper.FindExePath(AppConfig.Program); //frameworkHandle.SendMessage(TestMessageLevel.Error, "Framework: Exe " + exe); //frameworkHandle.SendMessage(TestMessageLevel.Error, "Framework: Cwd " + cwd); //frameworkHandle.SendMessage(TestMessageLevel.Error, "Framework: ResultFile " + resultFile); ProcessStartInfo info = new ProcessStartInfo() { Arguments = string.Format("{0} --resultJsonOutputFile \"{1}\" --specs \"{2}\"", AppConfig.Arguments, resultFile, test.CodeFilePath), FileName = exe, WorkingDirectory = cwd,//runContext.SolutionDirectory, UseShellExecute = false, CreateNoWindow = true, RedirectStandardError = true, RedirectStandardOutput = true, }; frameworkHandle.SendMessage(TestMessageLevel.Informational, $"Framework: Starting {exe} on '{cwd}' with arguments: {info.Arguments}"); Process p = new Process { StartInfo = info }; using (AutoResetEvent outputWaitHandle = new AutoResetEvent(false)) using (AutoResetEvent errorWaitHandle = new AutoResetEvent(false)) { p.OutputDataReceived += (sender, e) => { if (e.Data == null) { outputWaitHandle.Set(); } else if (!String.IsNullOrWhiteSpace(e.Data)) { // Console.WriteLine(e.Data); frameworkHandle.SendMessage(TestMessageLevel.Informational, e.Data); } }; p.ErrorDataReceived += (sender, e) => { if (e.Data == null) { errorWaitHandle.Set(); } else if (!string.IsNullOrWhiteSpace(e.Data)) { // Console.Error.WriteLine(e.Data); frameworkHandle.SendMessage(TestMessageLevel.Error, e.Data); } }; p.Start(); p.BeginOutputReadLine(); p.BeginErrorReadLine(); p.WaitForExit(); outputWaitHandle.WaitOne(10000); errorWaitHandle.WaitOne(10000); // Process completed. Check process.ExitCode here. frameworkHandle.SendMessage(TestMessageLevel.Informational, "Framework: Complete. Exit code: " + p.ExitCode.ToString()); } return(resultFile); }
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { }
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { var explicitlyInclude = CommonStringPrefix.Of(tests.Select(t => t.FullyQualifiedName)); InternalRunTests(tests, runContext, frameworkHandle, explicitlyInclude); }