// NOTE: an earlier version of this code had a FilterBuilder // property. This seemed to make sense, because we instantiate // it in two different places. However, the existence of an // NUnitTestFilterBuilder, containing a reference to an engine // service caused our second-level tests of the test executor // to throw an exception. So if you consider doing this, beware! #endregion #region ITestExecutor Implementation /// <summary> /// Called by the Visual Studio IDE to run all tests. Also called by TFS Build /// to run either all or selected tests. In the latter case, a filter is provided /// as part of the run context. /// </summary> /// <param name="sources">Sources to be run.</param> /// <param name="runContext">Context to use when executing the tests.</param> /// <param name="frameworkHandle">Test log to send results and messages through</param> public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { #if LAUNCHDEBUGGER if (!Debugger.IsAttached) Debugger.Launch(); #endif Initialize(runContext, frameworkHandle); try { foreach (var source in sources) { var assemblyName = source; if (!Path.IsPathRooted(assemblyName)) assemblyName = Path.Combine(Environment.CurrentDirectory, assemblyName); TestLog.Info("Running all tests in " + assemblyName); RunAssembly(assemblyName, TestFilter.Empty); } } catch (Exception ex) { if (ex is TargetInvocationException) ex = ex.InnerException; TestLog.Error("Exception thrown executing tests", ex); } finally { TestLog.Info(string.Format("NUnit Adapter {0}: Test execution complete", AdapterVersion)); Unload(); } }
/// <summary> /// Runs the tests. /// </summary> /// <param name="tests">Which tests should be run.</param> /// <param name="context">Context in which to run tests.</param> /// <param param name="framework">Where results should be stored.</param> public void RunTests(IEnumerable<TestCase> tests, IRunContext context, IFrameworkHandle framework) { _state = ExecutorState.Running; foreach (var test in tests) { if (_state == ExecutorState.Cancelling) { _state = ExecutorState.Cancelled; return; } try { var reportDocument = RunOrDebugCatchTest(test.Source, test.FullyQualifiedName, context, framework); var result = GetTestResultFromReport(test, reportDocument, framework); framework.RecordResult(result); } catch (Exception ex) { // Log it and move on. It will show up to the user as a test that hasn't been run. framework.SendMessage(TestMessageLevel.Error, "Exception occured when processing test case: " + test.FullyQualifiedName); framework.SendMessage(TestMessageLevel.Informational, "Message: " + ex.Message + "\nStacktrace:" + ex.StackTrace); } } }
public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { this.frameworkHandle = frameworkHandle; var testLogger = new TestLogger(frameworkHandle); testLogger.SendMainMessage("Execution started"); foreach (var group in tests.GroupBy(t => t.Source)) { testLogger.SendInformationalMessage(String.Format("Running selected: '{0}'", group.Key)); try { using (var sandbox = new Sandbox<Executor>(group.Key)) { var assemblyDirectory = new DirectoryInfo(Path.GetDirectoryName(group.Key)); Directory.SetCurrentDirectory(assemblyDirectory.FullName); sandbox.Content.Execute(this, group.Select(t => t.FullyQualifiedName).ToArray()); } } catch (Exception ex) { testLogger.SendErrorMessage(ex, String.Format("Exception found while executing tests in group '{0}'", group.Key)); // just go on with the next } } testLogger.SendMainMessage("Execution finished"); }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { //Debugger.Launch(); frameworkHandle.SendMessage(TestMessageLevel.Informational, Strings.EXECUTOR_STARTING); Settings settings = GetSettings(runContext); foreach (string currentAsssembly in sources.Distinct()) { try { if (!File.Exists(Path.Combine(Path.GetDirectoryName(Path.GetFullPath(currentAsssembly)),"Machine.Specifications.dll"))) { frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("Machine.Specifications.dll not found for {0}", currentAsssembly)); continue; } frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format(Strings.EXECUTOR_EXECUTINGIN, currentAsssembly)); this.executor.RunAssembly(currentAsssembly, settings, uri, frameworkHandle); } catch (Exception ex) { frameworkHandle.SendMessage(TestMessageLevel.Error, String.Format(Strings.EXECUTOR_ERROR, currentAsssembly, ex.Message)); } } frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("Complete on {0} assemblies ", sources.Count())); }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { SetupExecutionPolicy(); IEnumerable<TestCase> tests = PowerShellTestDiscoverer.GetTests(sources, null); RunTests(tests, runContext, frameworkHandle); }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { Guard.ArgumentNotNull("sources", sources); Guard.ArgumentNotNull("runContext", runContext); Guard.ArgumentNotNull("frameworkHandle", frameworkHandle); var cleanupList = new List<ExecutorWrapper>(); try { RemotingUtility.CleanUpRegisteredChannels(); cancelled = false; foreach (string source in sources) if (VsTestRunner.IsXunitTestAssembly(source)) RunTestsInAssembly(cleanupList, source, frameworkHandle); } finally { Thread.Sleep(1000); foreach (var executorWrapper in cleanupList) executorWrapper.Dispose(); } }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { this.frameworkHandle = frameworkHandle; var testLogger = new TestLogger(frameworkHandle); testLogger.SendMainMessage("Execution started"); foreach (var source in sources) { try { using (var sandbox = new Sandbox<Executor>(source)) { testLogger.SendInformationalMessage(String.Format("Running: '{0}'", source)); var assemblyDirectory = new DirectoryInfo(Path.GetDirectoryName(source)); Directory.SetCurrentDirectory(assemblyDirectory.FullName); sandbox.Content.Execute(this); } } catch (Exception ex) { testLogger.SendErrorMessage(ex, String.Format("Exception found while executing tests in source '{0}'", source)); // just go on with the next } } testLogger.SendMainMessage("Execution finished"); }
/// <summary> /// Called by the Visual Studio IDE to run all tests. Also called by TFS Build /// to run either all or selected tests. In the latter case, a filter is provided /// as part of the run context. /// </summary> /// <param name="sources">Sources to be run.</param> /// <param name="runContext">Context to use when executing the tests.</param> /// <param name="frameworkHandle">Test log to send results and messages through</param> public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { testLog.Initialize(frameworkHandle); Info("executing tests", "started"); try { // Ensure any channels registered by other adapters are unregistered CleanUpRegisteredChannels(); var tfsfilter = new TFSTestFilter(runContext); testLog.SendDebugMessage("Keepalive:" + runContext.KeepAlive); if (!tfsfilter.HasTfsFilterValue && runContext.KeepAlive) frameworkHandle.EnableShutdownAfterTestRun = true; foreach (var source in sources) { using (currentRunner = new AssemblyRunner(testLog, source, tfsfilter)) { currentRunner.RunAssembly(frameworkHandle); } currentRunner = null; } } catch (Exception ex) { testLog.SendErrorMessage("Exception " + ex); } finally { Info("executing tests", "finished"); } }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { IMessageLogger log = frameworkHandle; log.Version(); HandlePoorVisualStudioImplementationDetails(runContext, frameworkHandle); foreach (var assemblyPath in sources) { try { if (AssemblyDirectoryContainsFixie(assemblyPath)) { log.Info("Processing " + assemblyPath); var listener = new VisualStudioListener(frameworkHandle, assemblyPath); using (var environment = new ExecutionEnvironment(assemblyPath)) { environment.RunAssembly(new Options(), listener); } } else { log.Info("Skipping " + assemblyPath + " because it is not a test assembly."); } } catch (Exception exception) { log.Error(exception); } } }
public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { frameworkHandle.SendMessage(TestMessageLevel.Informational, Strings.EXECUTOR_STARTING); int executedSpecCount = 0; string currentAsssembly = string.Empty; try { ISpecificationExecutor specificationExecutor = this.adapterFactory.CreateExecutor(); IEnumerable<IGrouping<string, TestCase>> groupBySource = tests.GroupBy(x => x.Source); foreach (IGrouping<string, TestCase> grouping in groupBySource) { currentAsssembly = grouping.Key; frameworkHandle.SendMessage(TestMessageLevel.Informational, string.Format(Strings.EXECUTOR_EXECUTINGIN, currentAsssembly)); specificationExecutor.RunAssemblySpecifications(currentAsssembly, MSpecTestAdapter.uri, runContext, frameworkHandle, grouping); executedSpecCount += grouping.Count(); } frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format(Strings.EXECUTOR_COMPLETE, executedSpecCount, groupBySource.Count())); } catch (Exception ex) { frameworkHandle.SendMessage(TestMessageLevel.Error, string.Format(Strings.EXECUTOR_ERROR, currentAsssembly, ex.Message)); } finally { } }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { ChutzpahTracer.TraceInformation("Begin Test Adapter Run Tests"); var settingsProvider = runContext.RunSettings.GetSettings(AdapterConstants.SettingsName) as ChutzpahAdapterSettingsProvider; var settings = settingsProvider != null ? settingsProvider.Settings : new ChutzpahAdapterSettings(); ChutzpahTracingHelper.Toggle(settings.EnabledTracing); var testOptions = new TestOptions { TestLaunchMode = runContext.IsBeingDebugged ? TestLaunchMode.Custom: settings.OpenInBrowser ? TestLaunchMode.FullBrowser: TestLaunchMode.HeadlessBrowser, CustomTestLauncher = runContext.IsBeingDebugged ? new VsDebuggerTestLauncher() : null, MaxDegreeOfParallelism = runContext.IsBeingDebugged ? 1 : settings.MaxDegreeOfParallelism, ChutzpahSettingsFileEnvironments = new ChutzpahSettingsFileEnvironments(settings.ChutzpahSettingsFileEnvironments) }; testOptions.CoverageOptions.Enabled = runContext.IsDataCollectionEnabled; var callback = new ParallelRunnerCallbackAdapter(new ExecutionCallback(frameworkHandle, runContext)); testRunner.RunTests(sources, testOptions, callback); ChutzpahTracer.TraceInformation("End Test Adapter Run Tests"); }
public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { //Debugger.Launch(); frameworkHandle.SendMessage(TestMessageLevel.Informational, Strings.EXECUTOR_STARTING); int executedSpecCount = 0; Settings settings = GetSettings(runContext); string currentAsssembly = string.Empty; try { foreach (IGrouping<string, TestCase> grouping in tests.GroupBy(x => x.Source)) { currentAsssembly = grouping.Key; frameworkHandle.SendMessage(TestMessageLevel.Informational, string.Format(Strings.EXECUTOR_EXECUTINGIN, currentAsssembly)); List<VisualStudioTestIdentifier> testsToRun = grouping.Select(test => test.ToVisualStudioTestIdentifier()).ToList(); this.executor.RunAssemblySpecifications(currentAsssembly, testsToRun, settings, uri, frameworkHandle); executedSpecCount += grouping.Count(); } frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format(Strings.EXECUTOR_COMPLETE, executedSpecCount, tests.GroupBy(x => x.Source).Count())); } catch (Exception ex) { frameworkHandle.SendMessage(TestMessageLevel.Error, string.Format(Strings.EXECUTOR_ERROR, currentAsssembly, ex.Message)); } finally { } }
/// <summary> /// Runs the tests. /// </summary> /// <param name="tests">Tests to be run.</param> /// <param name="runContext">Context to use when executing the tests.</param> /// <param param name="frameworkHandle">Handle to the framework to record results and to do framework operations.</param> public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { m_cancelled = false; try { foreach (TestCase test in tests) { if (m_cancelled) { break; } frameworkHandle.RecordStart(test); frameworkHandle.SendMessage(TestMessageLevel.Informational, "Starting external test for " + test.DisplayName); var testOutcome = RunExternalTest(test, runContext, frameworkHandle, test); frameworkHandle.RecordResult(testOutcome); frameworkHandle.SendMessage(TestMessageLevel.Informational, "Test result:" + testOutcome.Outcome.ToString()); } } catch(Exception e) { frameworkHandle.SendMessage(TestMessageLevel.Error, "Exception during test execution: " +e.Message); } }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { List<TestCase> tests = new List<TestCase>(); TestDiscoverer.VisitTests(sources, t => tests.Add(t)); InternalRunTests(tests, runContext, frameworkHandle, null); }
private void RunTests(string source, IRunContext runContext, IFrameworkHandle frameworkHandle) { foreach (var result in ExternalTestExecutor.GetTestResults(source, null).Select(c => CreateTestResult(source, c))) { frameworkHandle.RecordStart(result.TestCase); frameworkHandle.RecordResult(result); frameworkHandle.RecordEnd(result.TestCase, result.Outcome); } }
public virtual void Debug(BoostTestRunnerCommandLineArgs args, BoostTestRunnerSettings settings, IFrameworkHandle framework) { Utility.Code.Require(settings, "settings"); using (Process process = Debug(framework, GetStartInfo(args, settings))) { MonitorProcess(process, settings.Timeout); } }
public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { ValidateArg.NotNull(tests, "tests"); ValidateArg.NotNull(runContext, "runContext"); ValidateArg.NotNull(frameworkHandle, "frameworkHandle"); _cancelRequested.Reset(); RunTestCases(tests, runContext, frameworkHandle); }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { RunTests(sources.Select(x => new TestCase(x, ExecutorDetails.Uri, x) { CodeFilePath = x, LineNumber = 2, DisplayName = "Abra cadabra" }), runContext, frameworkHandle); }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { var validSources = from source in sources where source.EndsWith(StringHelper.GetSearchExpression(), StringComparison.CurrentCultureIgnoreCase) select source; foreach (var source in validSources) { RunTest(frameworkHandle, source); } }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { //Debugger.Launch(); // this a temporary hack until I can figure out why running the specs per assembly directly using mspec does not work with a large number of specifications // when they are run diectly the first 100 or so specs run fine and then an error occurs saying it has taken more than 10 seconds and is being stopped // for now we just rediscover and run them like that, makes no sense TestCaseCollector collector = new TestCaseCollector(); this.DiscoverTests(sources, runContext, frameworkHandle, collector); this.RunTests(collector.TestCases, runContext, frameworkHandle); }
private void RunTests(IRunContext runContext, IFrameworkHandle testExecutionRecorder) { if (runContext.InIsolation) launcher.TestProject.TestRunnerFactoryName = StandardTestRunnerFactoryNames.IsolatedAppDomain; var extension = new VSTestWindowExtension(testExecutionRecorder, testCaseFactory, testResultFactory); launcher.TestProject.AddTestRunnerExtension(extension); launcher.Run(); }
public TestGeneratorAdapterTests() { testGeneratorDiscoverer = Substitute.For<ITestGeneratorDiscoverer>(); discoveryContext = Substitute.For<IDiscoveryContext>(); discoverySink = Substitute.For<ITestCaseDiscoverySink>(); logger = Substitute.For<IMessageLogger>(); frameworkHandle = Substitute.For<IFrameworkHandle>(); runContext = Substitute.For<IRunContext>(); testGenerator = Substitute.For<ITestGenerator>(); }
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle testExecutionRecorder) { launcher = new TestLauncher(); foreach (var source in sources) { launcher.AddFilePattern(source); } RunTests(runContext, testExecutionRecorder); }
public void RunAssembly(string source, Settings settings, Uri executorUri, IFrameworkHandle frameworkHandle) { source = Path.GetFullPath(source); using (var scope = new IsolatedAppDomainExecutionScope<AppDomainExecutor>(source)) { VSProxyAssemblySpecificationRunListener listener = new VSProxyAssemblySpecificationRunListener(source, frameworkHandle, executorUri, settings); AppDomainExecutor executor = scope.CreateInstance(); executor.RunAllTestsInAssembly(source, listener); } }
public TestExecutorSinkAdapter(IFrameworkHandle frameworkHandle, IEnumerable<TestCase> tests) { _frameworkHandle = frameworkHandle; _tests = new Dictionary<string, TestCase>(); foreach(var test in tests) { _tests[test.FullyQualifiedName] = test; } }
public VSTracker(IFrameworkHandle frameworkHandle, ITestSuite suite) { _frameworkHandle = frameworkHandle; _expectedTests = new Dictionary<string, TestCase>(); foreach (var testCase in suite.TestCases) { foreach (var test in testCase.Tests) { _expectedTests.Add(createDictionaryKey(test), test.ToVSTest()); } } }
public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle testExecutionRecorder) { launcher = new TestLauncher(); foreach (var test in tests) { launcher.AddFilePattern(test.Source); } SetTestFilter(tests); RunTests(runContext, testExecutionRecorder); }
public void RunAssembly(string source, Uri uri, IRunContext runContext, IFrameworkHandle frameworkHandle) { source = Path.GetFullPath(source); if (!File.Exists(source)) { throw new ArgumentException("Could not find file: " + source); } string assemblyFilename = source; string defaultConfigFile = SpecificationExecutor.GetDefaultConfigFile(source); runManager = new MSpecVSRunnerManager(); runManager.RunAllTestsInAssembly(assemblyFilename, defaultConfigFile, frameworkHandle, uri); }
public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { this.frameworkHandle = frameworkHandle; foreach (var group in tests.GroupBy(t => t.Source)) { frameworkHandle.SendMessage(TestMessageLevel.Informational, "Running selected: " + group.Key); using (var sandbox = new Sandbox<Executor>(group.Key)) { sandbox.Content.Execute(this, group.Select(t => t.FullyQualifiedName).ToArray()); } } }
public void RunAssemblySpecifications(string source, Uri uri, IRunContext runContext, IFrameworkHandle frameworkHandle, IEnumerable<TestCase> specifications) { source = Path.GetFullPath(source); if (!File.Exists(source)) { throw new ArgumentException("Could not find file: " + source); } string assemblyFilename = source; string defaultConfigFile = SpecificationExecutor.GetDefaultConfigFile(source); IEnumerable<string> specsToRun = specifications.Select(x => x.FullyQualifiedName).ToList(); runManager = new MSpecVSRunnerManager(); runManager.RunTestsInAssembly(assemblyFilename, defaultConfigFile, frameworkHandle, specsToRun, uri); }
public void RunTests(IEnumerable <string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { RunTestsWithSourcesCallback?.Invoke(sources, runContext, frameworkHandle); }
public void RunTests(IEnumerable <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { // We'll just punt and run everything in each file that contains the selected tests var sources = tests.Select(test => test.Source).Distinct(); RunTests(sources, runContext, frameworkHandle); }
/// <summary> /// Execute the parameter tests present in parameter source /// </summary> /// <param name="tests">Tests to execute.</param> /// <param name="runContext">The run context.</param> /// <param name="frameworkHandle">Handle to record test start/end/results.</param> /// <param name="source">The test container for the tests.</param> /// <param name="isDeploymentDone">Indicates if deployment is done.</param> private void ExecuteTestsInSource(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle, string source, bool isDeploymentDone) { Debug.Assert(!string.IsNullOrEmpty(source), "Source cannot be empty"); source = isDeploymentDone ? Path.Combine( PlatformServiceProvider.Instance.TestDeployment.GetDeploymentDirectory(), Path.GetFileName(source)) : source; using (var isolationHost = PlatformServiceProvider.Instance.CreateTestSourceHost(source, runContext?.RunSettings, frameworkHandle)) { var testRunner = isolationHost.CreateInstanceForType( typeof(UnitTestRunner), new object[] { MSTestSettings.CurrentSettings }) as UnitTestRunner; PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Created unit-test runner {0}", source); this.ExecuteTestsWithTestRunner(tests, runContext, frameworkHandle, source, testRunner); PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo( "Executed tests belonging to source {0}", source); } }
private void RunTestCases(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { // May be null, but this is handled by RunTestCase if it matters. // No VS instance just means no debugging, but everything else is // okay. using (var app = VisualStudioApp.FromEnvironmentVariable(NodejsConstants.NodeToolsProcessIdEnvironmentVariable)) { // .njsproj file path -> project settings var sourceToSettings = new Dictionary <string, NodejsProjectSettings>(); foreach (var test in tests) { if (_cancelRequested.WaitOne(0)) { break; } try { RunTestCase(app, frameworkHandle, runContext, test, sourceToSettings); } catch (Exception ex) { frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString()); } } } }
protected abstract void InvokeExecutor(LazyExtension <ITestExecutor, ITestExecutorCapabilities> executor, Tuple <Uri, string> executorUriExtensionTuple, RunContext runContext, IFrameworkHandle frameworkHandle);
public ExecutionCallback(IFrameworkHandle frameworkHandle, IRunContext runContext) { this.frameworkHandle = frameworkHandle; this.runContext = runContext; }
void ITestExecutor.RunTests(IEnumerable <string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { Guard.ArgumentNotNull("sources", sources); var stopwatch = Stopwatch.StartNew(); var logger = new LoggerHelper(frameworkHandle, stopwatch); // In this case, we need to go thru the files manually if (ContainsAppX(sources)) { #if PLATFORM_DOTNET var sourcePath = Windows.ApplicationModel.Package.Current.InstalledLocation.Path; #else var sourcePath = Environment.CurrentDirectory; #endif sources = Directory.GetFiles(sourcePath, "*.dll") .Where(file => !platformAssemblies.Contains(Path.GetFileName(file))) .ToList(); } RunTests(runContext, frameworkHandle, logger, () => GetTests(sources, logger, runContext)); }
void ITestExecutor.RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { Guard.ArgumentNotNull("tests", tests); Guard.ArgumentValid("tests", "AppX not supported in this overload", !ContainsAppX(tests.Select(t => t.Source))); var stopwatch = Stopwatch.StartNew(); var logger = new LoggerHelper(frameworkHandle, stopwatch); RunTests( runContext, frameworkHandle, logger, () => tests.GroupBy(testCase => testCase.Source) .Select(group => new AssemblyRunInfo { AssemblyFileName = group.Key, Configuration = LoadConfiguration(group.Key), TestCases = group.ToList() }) .ToList() ); }
private void RunTestCase( IFrameworkHandle frameworkHandle, IRunContext runContext, TestCase test, Dictionary <string, PythonProjectSettings> sourceToSettings ) { var testResult = new TestResult(test); frameworkHandle.RecordStart(test); testResult.StartTime = DateTimeOffset.Now; PythonProjectSettings settings; if (!sourceToSettings.TryGetValue(test.Source, out settings)) { sourceToSettings[test.Source] = settings = LoadProjectSettings(test.Source, _interpreterService); } if (settings == null) { frameworkHandle.SendMessage( TestMessageLevel.Error, "Unable to determine interpreter to use for " + test.Source); RecordEnd( frameworkHandle, test, testResult, null, "Unable to determine interpreter to use for " + test.Source, TestOutcome.Failed); return; } var debugMode = PythonDebugMode.None; if (runContext.IsBeingDebugged && _app != null) { debugMode = settings.EnableNativeCodeDebugging ? PythonDebugMode.PythonAndNative : PythonDebugMode.PythonOnly; } var testCase = new PythonTestCase(settings, test, debugMode); var dte = _app != null?_app.GetDTE() : null; if (dte != null && debugMode != PythonDebugMode.None) { dte.Debugger.DetachAll(); } if (!File.Exists(settings.Factory.Configuration.InterpreterPath)) { frameworkHandle.SendMessage(TestMessageLevel.Error, "Interpreter path does not exist: " + settings.Factory.Configuration.InterpreterPath); return; } var env = new Dictionary <string, string>(); var pythonPathVar = settings.Factory.Configuration.PathEnvironmentVariable; var pythonPath = testCase.SearchPaths; if (!string.IsNullOrWhiteSpace(pythonPathVar)) { if (_app != null) { var settingsManager = SettingsManagerCreator.GetSettingsManager(dte); if (settingsManager != null) { var store = settingsManager.GetReadOnlySettingsStore(SettingsScope.UserSettings); if (store != null && store.CollectionExists(@"PythonTools\Options\General")) { var settingStr = store.GetString(@"PythonTools\Options\General", "ClearGlobalPythonPath", "True"); bool settingBool; if (bool.TryParse(settingStr, out settingBool) && !settingBool) { pythonPath += ";" + Environment.GetEnvironmentVariable(pythonPathVar); } } } } env[pythonPathVar] = pythonPath; } foreach (var envVar in testCase.Environment) { env[envVar.Key] = envVar.Value; } using (var proc = ProcessOutput.Run( !settings.IsWindowsApplication ? settings.Factory.Configuration.InterpreterPath : settings.Factory.Configuration.WindowsInterpreterPath, testCase.Arguments, testCase.WorkingDirectory, env, false, null )) { bool killed = false; #if DEBUG frameworkHandle.SendMessage(TestMessageLevel.Informational, "cd " + testCase.WorkingDirectory); frameworkHandle.SendMessage(TestMessageLevel.Informational, "set " + (pythonPathVar ?? "") + "=" + (pythonPath ?? "")); frameworkHandle.SendMessage(TestMessageLevel.Informational, proc.Arguments); #endif proc.Wait(TimeSpan.FromMilliseconds(500)); if (debugMode != PythonDebugMode.None) { if (proc.ExitCode.HasValue) { // Process has already exited frameworkHandle.SendMessage(TestMessageLevel.Error, "Failed to attach debugger because the process has already exited."); if (proc.StandardErrorLines.Any()) { frameworkHandle.SendMessage(TestMessageLevel.Error, "Standard error from Python:"); foreach (var line in proc.StandardErrorLines) { frameworkHandle.SendMessage(TestMessageLevel.Error, line); } } } try { if (debugMode == PythonDebugMode.PythonOnly) { string qualifierUri = string.Format("tcp://{0}@localhost:{1}", testCase.DebugSecret, testCase.DebugPort); while (!_app.AttachToProcess(proc, PythonRemoteDebugPortSupplierUnsecuredId, qualifierUri)) { if (proc.Wait(TimeSpan.FromMilliseconds(500))) { break; } } } else { var engines = new[] { PythonDebugEngineGuid, VSConstants.DebugEnginesGuids.NativeOnly_guid }; while (!_app.AttachToProcess(proc, engines)) { if (proc.Wait(TimeSpan.FromMilliseconds(500))) { break; } } } #if DEBUG } catch (COMException ex) { frameworkHandle.SendMessage(TestMessageLevel.Error, "Error occurred connecting to debuggee."); frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString()); try { proc.Kill(); } catch (InvalidOperationException) { // Process has already exited } killed = true; } #else } catch (COMException) {
public void RunTests(IEnumerable <VsTestCase> vsTestCasesToRun, IRunContext runContext, IFrameworkHandle frameworkHandle) { try { TryRunTests(vsTestCasesToRun, runContext, frameworkHandle); } catch (Exception e) { _logger.LogError("Exception while running tests: " + e); } CommonFunctions.ReportErrors(_logger, "test execution", _settings.DebugMode); }
public void RunTests(IEnumerable <string> executables, IRunContext runContext, IFrameworkHandle frameworkHandle) { try { TryRunTests(executables, runContext, frameworkHandle); } catch (Exception e) { _logger.LogError($"Exception while running tests: {e}"); } CommonFunctions.ReportErrors(_logger, "test execution", _settings.DebugMode); }
private void DoRunTests(ICollection <TestCase> testCasesToRun, IRunContext runContext, IFrameworkHandle frameworkHandle) { if (testCasesToRun.Count == 0) { return; } bool isRunningInsideVisualStudio = !string.IsNullOrEmpty(runContext.SolutionDirectory); var reporter = new VsTestFrameworkReporter(frameworkHandle, isRunningInsideVisualStudio, _logger); var debuggerAttacher = _debuggerAttacher ?? new MessageBasedDebuggerAttacher(_settings.DebuggingNamedPipeId, _logger); var processExecutorFactory = new DebuggedProcessExecutorFactory(frameworkHandle, debuggerAttacher); lock (_lock) { if (_canceled) { return; } _executor = new GoogleTestExecutor(_logger, _settings, processExecutorFactory); } _executor.RunTests(testCasesToRun, reporter, runContext.IsBeingDebugged); reporter.AllTestsFinished(); }
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { throw new NotImplementedException(); }
static void RunTests(IMessageLogger log, IFrameworkHandle frameworkHandle, string assemblyPath, Action <NamedPipeServerStream> sendCommand) { if (!IsTestAssembly(assemblyPath)) { log.Info("Skipping " + assemblyPath + " because it is not a test assembly."); return; } log.Info("Processing " + assemblyPath); var pipeName = Guid.NewGuid().ToString(); Environment.SetEnvironmentVariable("FIXIE_NAMED_PIPE", pipeName); using (var pipe = new NamedPipeServerStream(pipeName, PipeDirection.InOut, 1, PipeTransmissionMode.Message)) using (var process = Start(assemblyPath, frameworkHandle)) { pipe.WaitForConnection(); sendCommand(pipe); var recorder = new ExecutionRecorder(frameworkHandle, assemblyPath); PipeMessage.CaseStarted lastCaseStarted = null; while (true) { var messageType = pipe.ReceiveMessage(); if (messageType == typeof(PipeMessage.CaseStarted).FullName) { var message = pipe.Receive <PipeMessage.CaseStarted>(); lastCaseStarted = message; recorder.Record(message); } else if (messageType == typeof(PipeMessage.CaseSkipped).FullName) { var testResult = pipe.Receive <PipeMessage.CaseSkipped>(); recorder.Record(testResult); } else if (messageType == typeof(PipeMessage.CasePassed).FullName) { var testResult = pipe.Receive <PipeMessage.CasePassed>(); recorder.Record(testResult); } else if (messageType == typeof(PipeMessage.CaseFailed).FullName) { var testResult = pipe.Receive <PipeMessage.CaseFailed>(); recorder.Record(testResult); } else if (messageType == typeof(PipeMessage.Exception).FullName) { var exception = pipe.Receive <PipeMessage.Exception>(); throw new RunnerException(exception); } else if (messageType == typeof(PipeMessage.Completed).FullName) { var completed = pipe.Receive <PipeMessage.Completed>(); break; } else if (!string.IsNullOrEmpty(messageType)) { var body = pipe.ReceiveMessage(); log.Error($"The test runner received an unexpected message of type {messageType}: {body}"); } else { var exception = new TestProcessExitException(process.TryGetExitCode()); if (lastCaseStarted != null) { recorder.Record(new PipeMessage.CaseFailed { Test = lastCaseStarted.Test, Name = lastCaseStarted.Name, Exception = new PipeMessage.Exception(exception) }); } throw exception; } } } }
protected abstract IEnumerable <Tuple <Uri, string> > GetExecutorUriExtensionMap(IFrameworkHandle testExecutorFrameworkHandle, RunContext runContext);
protected override IEnumerable <Tuple <Uri, string> > GetExecutorUriExtensionMap(IFrameworkHandle testExecutorFrameworkHandle, RunContext runContext) { this.executorUriVsTestList = this.GetExecutorVsTestCaseList(this.testCases); Debug.Assert(this.TestExecutionContext.TestCaseFilter == null, "TestCaseFilter should be null for specific tests."); runContext.FilterExpressionWrapper = null; return(this.executorUriVsTestList.Keys); }
private void RunTestCase(VisualStudioApp app, IFrameworkHandle frameworkHandle, IRunContext runContext, TestCase test, Dictionary <string, NodejsProjectSettings> sourceToSettings) { var testResult = new TestResult(test); frameworkHandle.RecordStart(test); testResult.StartTime = DateTimeOffset.Now; NodejsProjectSettings settings; if (!sourceToSettings.TryGetValue(test.Source, out settings)) { sourceToSettings[test.Source] = settings = LoadProjectSettings(test.Source); } if (settings == null) { frameworkHandle.SendMessage( TestMessageLevel.Error, "Unable to determine interpreter to use for " + test.Source); RecordEnd( frameworkHandle, test, testResult, null, "Unable to determine interpreter to use for " + test.Source, TestOutcome.Failed); return; } NodejsTestInfo testInfo = new NodejsTestInfo(test.FullyQualifiedName); List <string> args = new List <string>(); int port = 0; if (runContext.IsBeingDebugged && app != null) { app.GetDTE().Debugger.DetachAll(); args.AddRange(GetDebugArgs(settings, out port)); } var workingDir = Path.GetDirectoryName(CommonUtils.GetAbsoluteFilePath(settings.WorkingDir, testInfo.ModulePath)); args.AddRange(GetInterpreterArgs(test, workingDir, settings.ProjectRootDir)); //Debug.Fail("attach debugger"); if (!File.Exists(settings.NodeExePath)) { frameworkHandle.SendMessage(TestMessageLevel.Error, "Interpreter path does not exist: " + settings.NodeExePath); return; } lock (_syncObject) { _nodeProcess = ProcessOutput.Run( settings.NodeExePath, args, workingDir, null, false, null, false); #if DEBUG frameworkHandle.SendMessage(TestMessageLevel.Informational, "cd " + workingDir); frameworkHandle.SendMessage(TestMessageLevel.Informational, _nodeProcess.Arguments); #endif _nodeProcess.Wait(TimeSpan.FromMilliseconds(500)); if (runContext.IsBeingDebugged && app != null) { try { //the '#ping=0' is a special flag to tell VS node debugger not to connect to the port, //because a connection carries the consequence of setting off --debug-brk, and breakpoints will be missed. string qualifierUri = string.Format("tcp://localhost:{0}#ping=0", port); while (!app.AttachToProcess(_nodeProcess, NodejsRemoteDebugPortSupplierUnsecuredId, qualifierUri)) { if (_nodeProcess.Wait(TimeSpan.FromMilliseconds(500))) { break; } } #if DEBUG } catch (COMException ex) { frameworkHandle.SendMessage(TestMessageLevel.Error, "Error occurred connecting to debuggee."); frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString()); KillNodeProcess(); } #else } catch (COMException) {
protected override void InvokeExecutor(LazyExtension <ITestExecutor, ITestExecutorCapabilities> executor, Tuple <Uri, string> executorUri, RunContext runContext, IFrameworkHandle frameworkHandle) { executor?.Value.RunTests(this.executorUriVsTestList[executorUri], runContext, frameworkHandle); }
/// <summary> /// Execute the parameter tests /// </summary> /// <param name="tests">Tests to execute.</param> /// <param name="runContext">The run context.</param> /// <param name="frameworkHandle">Handle to record test start/end/results.</param> /// <param name="isDeploymentDone">Indicates if deployment is done.</param> internal virtual void ExecuteTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle, bool isDeploymentDone) { var testsBySource = from test in tests group test by test.Source into testGroup select new { Source = testGroup.Key, Tests = testGroup }; foreach (var group in testsBySource) { this.ExecuteTestsInSource(group.Tests, runContext, frameworkHandle, group.Source, isDeploymentDone); } }
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { RunTestsWithTestsCallback?.Invoke(tests, runContext, frameworkHandle); }
public DebuggedProcessLauncher(IFrameworkHandle handle) { _frameworkHandle = handle; }
public IEnumerable <Tuple <Uri, string> > CallGetExecutorUriExtensionMap( IFrameworkHandle testExecutorFrameworkHandle, RunContext runContext) { return(this.GetExecutorUriExtensionMap(testExecutorFrameworkHandle, runContext)); }
public void RunTests(IEnumerable <string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { throw new NotImplementedException(); }
private void RunTestCases(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle, NodejsProjectSettings settings) { // May be null, but this is handled by RunTestCase if it matters. // No VS instance just means no debugging, but everything else is // okay. if (tests.Count() == 0) { return; } using (var app = VisualStudioApp.FromEnvironmentVariable(NodejsConstants.NodeToolsProcessIdEnvironmentVariable)) { var port = 0; var nodeArgs = new List <string>(); // .njsproj file path -> project settings var sourceToSettings = new Dictionary <string, NodejsProjectSettings>(); var testObjects = new List <TestCaseObject>(); if (!File.Exists(settings.NodeExePath)) { frameworkHandle.SendMessage(TestMessageLevel.Error, "Interpreter path does not exist: " + settings.NodeExePath); return; } // All tests being run are for the same test file, so just use the first test listed to get the working dir var testInfo = new NodejsTestInfo(tests.First().FullyQualifiedName); var workingDir = Path.GetDirectoryName(CommonUtils.GetAbsoluteFilePath(settings.WorkingDir, testInfo.ModulePath)); var nodeVersion = Nodejs.GetNodeVersion(settings.NodeExePath); // We can only log telemetry when we're running in VS. // Since the required assemblies are not on disk if we're not running in VS, we have to reference them in a separate method // this way the .NET framework only tries to load the assemblies when we actually need them. if (app != null) { LogTelemetry(tests.Count(), nodeVersion, runContext.IsBeingDebugged); } foreach (var test in tests) { if (_cancelRequested.WaitOne(0)) { break; } if (settings == null) { frameworkHandle.SendMessage( TestMessageLevel.Error, $"Unable to determine interpreter to use for {test.Source}."); frameworkHandle.RecordEnd(test, TestOutcome.Failed); } var args = new List <string>(); args.AddRange(GetInterpreterArgs(test, workingDir, settings.ProjectRootDir)); // Fetch the run_tests argument for starting node.exe if not specified yet if (nodeArgs.Count == 0 && args.Count > 0) { nodeArgs.Add(args[0]); } testObjects.Add(new TestCaseObject(args[1], args[2], args[3], args[4], args[5])); } if (runContext.IsBeingDebugged && app != null) { app.GetDTE().Debugger.DetachAll(); // Ensure that --debug-brk is the first argument nodeArgs.InsertRange(0, GetDebugArgs(out port)); } _nodeProcess = ProcessOutput.Run( settings.NodeExePath, nodeArgs, settings.WorkingDir, /* env */ null, /* visible */ false, /* redirector */ new TestExecutionRedirector(this.ProcessTestRunnerEmit), /* quote args */ false); if (runContext.IsBeingDebugged && app != null) { try { //the '#ping=0' is a special flag to tell VS node debugger not to connect to the port, //because a connection carries the consequence of setting off --debug-brk, and breakpoints will be missed. var qualifierUri = string.Format("tcp://localhost:{0}#ping=0", port); while (!app.AttachToProcess(_nodeProcess, NodejsRemoteDebugPortSupplierUnsecuredId, qualifierUri)) { if (_nodeProcess.Wait(TimeSpan.FromMilliseconds(500))) { break; } } #if DEBUG } catch (COMException ex) { frameworkHandle.SendMessage(TestMessageLevel.Error, "Error occurred connecting to debuggee."); frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString()); KillNodeProcess(); } #else } catch (COMException) {
public void CallInvokeExecutor(LazyExtension <ITestExecutor, ITestExecutorCapabilities> executor, Tuple <Uri, string> executorUriExtensionTuple, RunContext runContext, IFrameworkHandle frameworkHandle) { this.InvokeExecutor(executor, executorUriExtensionTuple, runContext, frameworkHandle); }
public void RunTests(IEnumerable <string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { frameworkHandle.SendMessage(TestMessageLevel.Informational, "Running sources directly"); }
/// <summary> /// Execute the parameter tests present in parameter source /// </summary> /// <param name="tests">Tests to execute.</param> /// <param name="runContext">The run context.</param> /// <param name="frameworkHandle">Handle to record test start/end/results.</param> /// <param name="source">The test container for the tests.</param> /// <param name="isDeploymentDone">Indicates if deployment is done.</param> private void ExecuteTestsInSource(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle, string source, bool isDeploymentDone) { Debug.Assert(!string.IsNullOrEmpty(source), "Source cannot be empty"); if (isDeploymentDone) { source = Path.Combine(PlatformServiceProvider.Instance.TestDeployment.GetDeploymentDirectory(), Path.GetFileName(source)); } using (var isolationHost = PlatformServiceProvider.Instance.CreateTestSourceHost(source, runContext?.RunSettings, frameworkHandle)) { // Create an instance of a type defined in adapter so that adapter gets loaded in the child app domain var testRunner = isolationHost.CreateInstanceForType( typeof(UnitTestRunner), new object[] { MSTestSettings.CurrentSettings }) as UnitTestRunner; // After loading adapter reset the chils-domain's appbase to point to test source location isolationHost.UpdateAppBaseToTestSourceLocation(); PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Created unit-test runner {0}", source); // Default test set is filtered tests based on user provided filter criteria IEnumerable <TestCase> testsToRun = Enumerable.Empty <TestCase>(); var filterExpression = this.TestMethodFilter.GetFilterExpression(runContext, frameworkHandle, out var filterHasError); if (filterHasError) { // Bail out without processing everything else below. return; } testsToRun = tests.Where(t => MatchTestFilter(filterExpression, t, this.TestMethodFilter)); // this is done so that appropriate values of testcontext properties are set at source level // and are merged with session level parameters var sourceLevelParameters = PlatformServiceProvider.Instance.SettingsProvider.GetProperties(source); if (this.sessionParameters != null && this.sessionParameters.Count > 0) { sourceLevelParameters = sourceLevelParameters.Concat(this.sessionParameters).ToDictionary(x => x.Key, x => x.Value); } var sourceSettingsProvider = isolationHost.CreateInstanceForType( typeof(TestAssemblySettingsProvider), null) as TestAssemblySettingsProvider; var sourceSettings = sourceSettingsProvider.GetSettings(source); var parallelWorkers = sourceSettings.Workers; var parallelScope = sourceSettings.Scope; if (MSTestSettings.CurrentSettings.ParallelizationWorkers.HasValue) { // The runsettings value takes precedence over an assembly level setting. Reset the level. parallelWorkers = MSTestSettings.CurrentSettings.ParallelizationWorkers.Value; } if (MSTestSettings.CurrentSettings.ParallelizationScope.HasValue) { // The runsettings value takes precedence over an assembly level setting. Reset the level. parallelScope = MSTestSettings.CurrentSettings.ParallelizationScope.Value; } if (!MSTestSettings.CurrentSettings.DisableParallelization && sourceSettings.CanParallelizeAssembly && parallelWorkers > 0) { // Parallelization is enabled. Let's do further classification for sets. var logger = (IMessageLogger)frameworkHandle; logger.SendMessage( TestMessageLevel.Informational, string.Format(CultureInfo.CurrentCulture, Resource.TestParallelizationBanner, source, parallelWorkers, parallelScope)); // Create test sets for execution, we can execute them in parallel based on parallel settings IEnumerable <IGrouping <bool, TestCase> > testsets = Enumerable.Empty <IGrouping <bool, TestCase> >(); // Parallel and not parallel sets. testsets = testsToRun.GroupBy(t => t.GetPropertyValue <bool>(TestAdapter.Constants.DoNotParallelizeProperty, false)); var parallelizableTestSet = testsets.FirstOrDefault(g => g.Key == false); var nonparallelizableTestSet = testsets.FirstOrDefault(g => g.Key == true); if (parallelizableTestSet != null) { ConcurrentQueue <IEnumerable <TestCase> > queue = null; // Chunk the sets into further groups based on parallel level switch (parallelScope) { case ExecutionScope.MethodLevel: queue = new ConcurrentQueue <IEnumerable <TestCase> >(parallelizableTestSet.Select(t => new[] { t })); break; case ExecutionScope.ClassLevel: queue = new ConcurrentQueue <IEnumerable <TestCase> >(parallelizableTestSet.GroupBy(t => t.GetPropertyValue(TestAdapter.Constants.TestClassNameProperty) as string)); break; } var tasks = new List <Task>(); for (int i = 0; i < parallelWorkers; i++) { tasks.Add(Task.Factory.StartNew( () => { while (!queue.IsEmpty) { if (this.cancellationToken != null && this.cancellationToken.Canceled) { // if a cancellation has been requested, do not queue any more test runs. break; } if (queue.TryDequeue(out IEnumerable <TestCase> testSet)) { this.ExecuteTestsWithTestRunner(testSet, runContext, frameworkHandle, source, sourceLevelParameters, testRunner); } } }, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default)); } Task.WaitAll(tasks.ToArray()); } // Queue the non parallel set if (nonparallelizableTestSet != null) { this.ExecuteTestsWithTestRunner(nonparallelizableTestSet, runContext, frameworkHandle, source, sourceLevelParameters, testRunner); } } else { this.ExecuteTestsWithTestRunner(testsToRun, runContext, frameworkHandle, source, sourceLevelParameters, testRunner); } this.RunCleanup(frameworkHandle, testRunner); PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo( "Executed tests belonging to source {0}", source); } }
private void TryRunTests(IEnumerable <VsTestCase> vsTestCasesToRun, IRunContext runContext, IFrameworkHandle frameworkHandle) { var stopwatch = StartStopWatchAndInitEnvironment(runContext, frameworkHandle); if (!AbleToRun(runContext)) { return; } var vsTestCasesToRunAsArray = vsTestCasesToRun as VsTestCase[] ?? vsTestCasesToRun.ToArray(); ISet <string> allTraitNames = GetAllTraitNames(vsTestCasesToRunAsArray.Select(tc => tc.ToTestCase())); var filter = new TestCaseFilter(runContext, allTraitNames, _logger); vsTestCasesToRun = filter.Filter(vsTestCasesToRunAsArray); ICollection <TestCase> testCasesToRun = vsTestCasesToRun.Select(tc => tc.ToTestCase()).ToArray(); DoRunTests(testCasesToRun, runContext, frameworkHandle); stopwatch.Stop(); _logger.LogInfo($"Google Test execution completed, overall duration: {stopwatch.Elapsed}."); }
private void TryRunTests(IEnumerable <string> executables, IRunContext runContext, IFrameworkHandle frameworkHandle) { var stopwatch = StartStopWatchAndInitEnvironment(runContext, frameworkHandle); if (!AbleToRun(runContext)) { return; } IList <TestCase> allTestCasesInExecutables = GetAllTestCasesInExecutables(executables).ToList(); ISet <string> allTraitNames = GetAllTraitNames(allTestCasesInExecutables); var filter = new TestCaseFilter(runContext, allTraitNames, _logger); List <VsTestCase> vsTestCasesToRun = filter.Filter(allTestCasesInExecutables.Select(tc => tc.ToVsTestCase())).ToList(); ICollection <TestCase> testCasesToRun = allTestCasesInExecutables.Where( tc => vsTestCasesToRun.Any(vtc => tc.FullyQualifiedName == vtc.FullyQualifiedName)).ToArray(); DoRunTests(testCasesToRun, runContext, frameworkHandle); stopwatch.Stop(); _logger.LogInfo($"Google Test execution completed, overall duration: {stopwatch.Elapsed}."); }
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { var CatchExe = tests.First().Source; var timer = Stopwatch.StartNew(); // Get a list of all test case names var listOfTestCases = tests.Aggregate("", (acc, test) => acc + test.DisplayName + "\n"); // Use the directory of the executable as the working directory. string workingDirectory = System.IO.Path.GetDirectoryName(CatchExe); if (workingDirectory == "") { workingDirectory = "."; } // Write them to the input file for Catch runner string caseFile = "test.cases"; System.IO.File.WriteAllText( workingDirectory + System.IO.Path.DirectorySeparatorChar + caseFile, listOfTestCases); string originalDirectory = Directory.GetCurrentDirectory(); // Execute the tests IList <string> output_text; string arguments = "-r xml --durations yes --input-file=" + caseFile; if (runContext.IsBeingDebugged) { output_text = ProcessRunner.RunDebugProcess(frameworkHandle, CatchExe, arguments, workingDirectory); } else { output_text = ProcessRunner.RunProcess(CatchExe, arguments, workingDirectory); } timer.Stop(); frameworkHandle.SendMessage(TestMessageLevel.Informational, "Overall time " + timer.Elapsed.ToString()); // Output as a single string. string output = output_text.Aggregate("", (acc, add) => acc + add); System.IO.MemoryStream reader = new System.IO.MemoryStream(System.Text.Encoding.ASCII.GetBytes(output)); var serializer = new XmlSerializer(typeof(CatchTestAdapter.Tests.Catch)); var catchResult = (CatchTestAdapter.Tests.Catch)serializer.Deserialize(reader); foreach (var testCase in catchResult.TestCases) { // Find the matching test case var test = tests.Where((test_case) => test_case.DisplayName == testCase.Name).ElementAt(0); var testResult = new TestResult(test); // Add the test execution time provided by Catch to the result. var testTime = testCase.Result.Duration; testResult.Duration = TimeSpan.FromSeconds(Double.Parse(testTime, CultureInfo.InvariantCulture)); if (testCase.Result.Success == "true") { testResult.Outcome = TestOutcome.Passed; } else { // Mark failure. testResult.Outcome = TestOutcome.Failed; // Parse the failure to a flat result. List <FlatResult> failures = GetFlatFailure(testCase); testResult.ErrorMessage = $"{Environment.NewLine}"; for (int i = 1; i <= failures.Count; ++i) { var failure = failures[i - 1]; // Populate the error message. var newline = failure.SectionPath.IndexOf("\n"); if (newline != -1) { // Remove first line of the SectionPath, which is the test case name. failure.SectionPath = failure.SectionPath.Substring(failure.SectionPath.IndexOf("\n") + 1); testResult.ErrorMessage += $"#{i} - {failure.SectionPath}{Environment.NewLine}{failure.Expression}{Environment.NewLine}"; } else { testResult.ErrorMessage += $"#{i} - {failure.Expression}{Environment.NewLine}"; } // And the error stack. testResult.ErrorStackTrace += $"at #{i} - {test.DisplayName}() in {failure.FilePath}:line {failure.LineNumber}{Environment.NewLine}"; } } // Finally record the result. frameworkHandle.RecordResult(testResult); } // Remove the temporary input file. System.IO.File.Delete(caseFile); }