public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext , IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { #if DEBUG && LAUNCH_DEBUGGER System.Diagnostics.Debugger.Launch(); #endif try { Logger.Initialize(logger); var sourceGroups = sources.GroupBy(Path.GetExtension); foreach (IGrouping<string, string> sourceGroup in sourceGroups) { IIutestTestDiscoverer discoverer = iutestTestDiscovererFactory.GetTestDiscoverer(sourceGroup.Key); if( discoverer != null ) { discoverer.DiscoverTests(sourceGroup, discoveryContext, logger, discoverySink); } } } catch( Exception e ) { Logger.Error("iutest_adapter: Exception caught while DiscoverTests: {0}, {1} ", e.Message, e.HResult ); Logger.Error(e.StackTrace); } finally { Logger.Uninitialize(); } }
private void ParseTests(StreamReader standardOutput, ITestCaseDiscoverySink discoverySink, IMessageLogger logger, string source) { string testcase = ""; bool testsStarted = false; Regex testCaseMatch = new Regex(@".*\.$"); while (standardOutput.Peek() != -1) { string line = standardOutput.ReadLine(); if (!string.IsNullOrEmpty(line)) { if (testCaseMatch.IsMatch(line)) { testsStarted = true; testcase = line.Trim(); logger.SendMessage(TestMessageLevel.Informational, string.Format("Found test case {0}", testcase.Trim('.'))); } else if(testsStarted) { TestCase test = new TestCase(testcase + line.Trim(), Plugin.VisualStudio2012.GTest.GTestExecutor.ExecutorUri, source); discoverySink.SendTestCase(test); logger.SendMessage(TestMessageLevel.Informational, string.Format("Found test {0}", testcase + line.Trim())); } } } }
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger log, ITestCaseDiscoverySink discoverySink) { log.Version(); RemotingUtility.CleanUpRegisteredChannels(); foreach (var assemblyPath in sources) { try { if (AssemblyDirectoryContainsFixie(assemblyPath)) { log.Info("Processing " + assemblyPath); using (var environment = new ExecutionEnvironment(assemblyPath)) { var methodGroups = environment.DiscoverTestMethodGroups(new Options()); foreach (var methodGroup in methodGroups) discoverySink.SendTestCase(new TestCase(methodGroup.FullName, VsTestExecutor.Uri, assemblyPath)); } } else { log.Info("Skipping " + assemblyPath + " because it is not a test assembly."); } } catch (Exception exception) { log.Error(exception); } } }
public static IEnumerable<TestCase> GetTests(IEnumerable<string> sourceFiles, ITestCaseDiscoverySink discoverySink) { var tests = new List<TestCase>(); Parallel.ForEach(sourceFiles, s => { var sourceCode = File.ReadAllText(s); var matches = TestFinderRegex.Matches(sourceCode); foreach (Match m in matches) { var methodParts = m.Groups["Method"].Value.Split(new string[] { ".prototype." }, System.StringSplitOptions.None); var testClass = methodParts[0]; var testMethod = methodParts[1]; var testName = m.Groups["Name"].Value; var testDescription = m.Groups["Description"].Value; var testCase = new TestCase(String.Join(".", methodParts), TSTestExecutor.ExecutorUri, s) { CodeFilePath = s, DisplayName = testName, }; if (discoverySink != null) { discoverySink.SendTestCase(testCase); } tests.Add(testCase); } }); return tests; }
public static List<TestCase> GetTests(IEnumerable<string> sources, ITestCaseDiscoverySink discoverySink, IMessageLogger logger = null) { var tests = new List<TestCase>(); foreach (var source in sources) { var testType = TestType.Pester; var scriptContents = System.IO.File.ReadAllText(source); if (scriptContents.StartsWith("#pester", StringComparison.OrdinalIgnoreCase)) { testType = TestType.Pester; } else if (scriptContents.StartsWith("#psate", StringComparison.OrdinalIgnoreCase)) { testType = TestType.PSate; } if (testType == TestType.Pester) { DiscoverPesterTests(discoverySink, logger, source, tests); } else if (testType == TestType.PSate) { DiscoverPsateTests(discoverySink, source, tests); } } return tests; }
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, ITestCaseDiscoverySink discoverySink) { Code.Require(sources, "sources"); Code.Require(discoverySink, "discoverySink"); if (this.Settings.DiscoveryMethodType == DiscoveryMethodType.DiscoveryListContent) { // Delegate to ListContentDiscoverer ListContentDiscoverer discoverer = new ListContentDiscoverer(new ExternalBoostTestRunnerFactory(), VSProvider); discoverer.DiscoverTests(sources, discoveryContext, discoverySink); } else { foreach (string source in sources) { TestFramework framework = DiscoverTestFramework(source); if ((framework != null) && (framework.MasterTestSuite != null)) { VSDiscoveryVisitor visitor = new VSDiscoveryVisitor(source, discoverySink); framework.MasterTestSuite.Apply(visitor); } } } }
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { testLogger = new TestLogger(logger); testLogger.SendMainMessage("Discovery started"); foreach (var source in sources) { testLogger.SendDebugMessage(String.Format("Processing: '{0}'", source)); try { using (var sandbox = new Sandbox<Discoverer>(source)) { if (sandbox.Content != null) { sandbox.Content .DiscoverTests() .Select(name => name.ToTestCase(source)) .ForEach(discoverySink.SendTestCase); } } } catch (Exception ex) { testLogger.SendErrorMessage(ex, String.Format("Exception found while discovering tests in source '{0}'", source)); // just go on with the next } } testLogger.SendMainMessage("Discovery finished"); }
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { GetTests(sources, discoverySink); }
private int ProcessTestCases(ITest test, ITestCaseDiscoverySink discoverySink, TestConverter testConverter) { int cases = 0; if (test.IsSuite) { foreach (ITest child in test.Tests) cases += ProcessTestCases(child, discoverySink,testConverter); } else { try { #if LAUNCHDEBUGGER Debugger.Launch(); #endif TestCase testCase = testConverter.ConvertTestCase(test); discoverySink.SendTestCase(testCase); cases += 1; } catch (System.Exception ex) { testLog.SendErrorMessage("Exception converting " + test.TestName.FullName, ex); } } return cases; }
private async Task<int> DiscoverTests(TestSourceSettings settings, ITestLogger logger, ITestCaseDiscoverySink discoverySink) { logger = new TestLogger(logger, settings.Name, "Discover"); var count = 0; var tests = new ConcurrentBag<Guid>(); if (settings.Port > 0) { logger.Debug("Start"); var discoverCommand = new DiscoverCommand(settings.Port); await discoverCommand.Run(spec => { var testCase = CreateTestCase(settings, spec); tests.Add(testCase.Id); discoverySink.SendTestCase(testCase); count += 1; }); await new RequestRunCommand(settings.Port).Run(tests); logger.Debug("Complete: {0} tests", count); } else { logger.Error("Not connected to {0}", TestAdapterInfo.Name); } return count; }
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { _logger = logger; _logger.SendMessage(TestMessageLevel.Informational, ">>> DiscoverTests"); GetTests(sources, discoverySink); }
internal static IEnumerable<TestCase> GetTests(IEnumerable<string> sources, ITestCaseDiscoverySink discoverySink) { //if(!Debugger.IsAttached) // Debugger.Launch(); var tests = new List<TestCase>(); foreach (string source in sources) { var TestNames = GetTestNameFromFile(source); foreach (var testName in TestNames) { var normalizedSource = source.ToLowerInvariant(); var testCase = new TestCase(testName.Key, ProtractorTestExecutor.ExecutorUri, normalizedSource); tests.Add(testCase); testCase.CodeFilePath = source; testCase.LineNumber = testName.Value; if (discoverySink != null) { discoverySink.SendTestCase(testCase); } } } return tests; }
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { ChutzpahTracer.TraceInformation("Begin Test Adapter Discover Tests"); var settingsProvider = discoveryContext.RunSettings.GetSettings(AdapterConstants.SettingsName) as ChutzpahAdapterSettingsProvider; var settings = settingsProvider != null ? settingsProvider.Settings : new ChutzpahAdapterSettings(); ChutzpahTracingHelper.Toggle(settings.EnabledTracing); var testOptions = new TestOptions { MaxDegreeOfParallelism = settings.MaxDegreeOfParallelism, ChutzpahSettingsFileEnvironments = new ChutzpahSettingsFileEnvironments(settings.ChutzpahSettingsFileEnvironments) }; IList<TestError> errors; var testCases = testRunner.DiscoverTests(sources, testOptions, out errors); ChutzpahTracer.TraceInformation("Sending discovered tests to test case discovery sink"); foreach (var testCase in testCases) { var vsTestCase = testCase.ToVsTestCase(); discoverySink.SendTestCase(vsTestCase); } foreach (var error in errors) { logger.SendMessage(TestMessageLevel.Error, RunnerCallback.FormatFileErrorMessage(error)); } ChutzpahTracer.TraceInformation("End Test Adapter Discover Tests"); }
public static IEnumerable<TestCase> Discover(IEnumerable<string> sources, ITestCaseDiscoverySink discoverySink) { List<TestCase> result = new List<TestCase>(); // System.Diagnostics.Debugger.Launch(); var specList = new List<dynamic>(); foreach (var source in sources) { using (var sandbox = new Sandbox<Discover>(source)) { List<DefinitionSource> discoveredDefinitions = new List<DefinitionSource>(); try { discoveredDefinitions = sandbox.Content.DiscoverSpecsFromCurrentAssembly(); } catch (Exception a) { Console.WriteLine(a.Message); } discoveredDefinitions .Select(x => AddToSink(source, x, discoverySink)) .ToList() .ForEach(x => result.Add(x)); } } return result; }
/// <summary> /// Finds tests in Catch unit test binaries. Note: We have to run the binary to enumerate tests. /// </summary> /// <param name="sources">Binaries to search for tests.</param> public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext context, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { currentLogger = logger; foreach (var source in sources) { switch (checkCatchTestBinaryFormat(source)) { case CatchBinaryFormat.NoCatchTestBinary: continue; case CatchBinaryFormat.CatchTestBinary: foreach (var testCase in ListTestsInBinary(source)) { discoverySink.SendTestCase(testCase); } continue; case CatchBinaryFormat.CatchTestBinaryWithXmlTestList: foreach (var testCase in ListTestsInBinaryXml(source)) { discoverySink.SendTestCase(testCase); } break; } } }
public virtual void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { TestAdapterInfo = CreateTestAdapterInfo(); var testLogger = TestAdapterInfo.CreateLogger(logger); var discoverLogger = new TestLogger(testLogger, "Discover"); var testSettings = discoveryContext.RunSettings.GetTestSettings(TestAdapterInfo.SettingsName); var count = 0; var testCount = 0; foreach (var source in sources) { var sourceSettings = GetSourceSettings(source, testSettings); if (sourceSettings != null) { testCount += DiscoverTests(sourceSettings, testLogger, discoverySink).Result; count += 1; } else { discoverLogger.Warn("Could not get settings for {0}", source); } } if (count > 0 || testCount > 0) { discoverLogger.Info("{0} tests discovered in {1} test containers", testCount, count); } }
internal static IList<TestCase> GetTests(IEnumerable<string> sources, ITestCaseDiscoverySink discoverySink = null, ITestContainer testContainer = null) { IList<TestCase> tests = new List<TestCase>(); foreach (var assemblyFileName in sources) { Assembly assembly = Assembly.LoadFrom(assemblyFileName); var methodsWithMoyaAttributes = assembly.GetTypes() .SelectMany(t => t.GetMethods()) .Where(Reflection.MethodInfoHasMoyaAttribute) .ToArray(); foreach (MethodInfo methodWithMoyaAttributes in methodsWithMoyaAttributes) { var testCase = new TestCase(methodWithMoyaAttributes.Name, Constants.ExecutorUri, assemblyFileName) { Id = Guid.NewGuid() }; tests.Add(testCase); if (discoverySink != null) { discoverySink.SendTestCase(testCase); } if (testContainer != null) { testContainer.AddTestCaseAndMethod(testCase, methodWithMoyaAttributes); } } } return tests; }
public void DiscoverTests(IEnumerable<string> sources , IDiscoveryContext discoveryContext , Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging.IMessageLogger logger , ITestCaseDiscoverySink discoverySink) { GetTests(sources, discoverySink); }
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { var validSources = from source in sources where source.EndsWith(StringHelper.GetSearchExpression(), StringComparison.CurrentCultureIgnoreCase) select source; foreach (var source in validSources) { var results = appDomainRunner.ExecuteSpecifications(source); var query = from result in results from @group in result.Examples from example in @group.Examples select example; foreach (var example in query) { var testCase = new TestCase(example.Reason, DefaultTestExecutor.ExecutorUri, source) { CodeFilePath = example.FileName, LineNumber = example.LineNumber }; discoverySink.SendTestCase(testCase); } } }
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { Code.Require(sources, "sources"); Code.Require(discoverySink, "discoverySink"); BoostTestAdapterSettings settings = BoostTestAdapterSettingsProvider.GetSettings(discoveryContext); _sourceFilters = SourceFilterFactory.Get(settings); IDictionary<string, ProjectInfo> solutioninfo = null; var numberOfAttempts = 100; // try several times to overcome "Application is Busy" COMException while (numberOfAttempts > 0) { try { solutioninfo = PrepareTestCaseData(sources); // set numberOfAttempts = 0, because there is no need to try again, // since obviously no exception was thrown at this point numberOfAttempts = 0; } catch (COMException) { --numberOfAttempts; // re-throw after all attempts have failed if (numberOfAttempts == 0) { throw; } } } GetBoostTests(solutioninfo, discoverySink); }
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { ValidateArg.NotNull(sources, "sources"); ValidateArg.NotNull(discoverySink, "discoverySink"); var settings = discoveryContext.RunSettings; DiscoverTests(sources, logger, discoverySink, settings); }
/// <summary> /// @todo add more metadata to test cases (however ctest alone does not provide everything needed) /// </summary> /// <param name="sources"></param> /// <param name="discoveryContext"></param> /// <param name="logger"></param> /// <param name="discoverySink"></param> public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging.IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { foreach (var source in sources) { var testcase = CTestCase.Parse(source); if (testcase == null) continue; discoverySink.SendTestCase(testcase); } }
/// <summary> /// Helper methods which adds a test case to an internal list and sends the test to the discovery sink /// </summary> /// <param name="testCase">the test case to be added</param> /// <param name="discoverySink">the discovery sink where the test case is sent to</param> public static void AddTestCase(TestCase testCase, ITestCaseDiscoverySink discoverySink) { //send to discovery sink if (null != discoverySink) { Logger.Info("Found test: {0}", testCase.FullyQualifiedName); discoverySink.SendTestCase(testCase); } }
public TestGeneratorAdapterTests() { testGeneratorDiscoverer = Substitute.For<ITestGeneratorDiscoverer>(); discoveryContext = Substitute.For<IDiscoveryContext>(); discoverySink = Substitute.For<ITestCaseDiscoverySink>(); logger = Substitute.For<IMessageLogger>(); frameworkHandle = Substitute.For<IFrameworkHandle>(); runContext = Substitute.For<IRunContext>(); testGenerator = Substitute.For<ITestGenerator>(); }
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger, ITestCaseDiscoverySink discoverySink) { testLog.Initialize(messageLogger); Info("discovering tests", "started"); // Ensure any channels registered by other adapters are unregistered CleanUpRegisteredChannels(); foreach (string sourceAssembly in sources) { testLog.SendDebugMessage("Processing " + sourceAssembly); TestRunner runner = new TestDomain(); TestPackage package = new TestPackage(sourceAssembly); TestConverter testConverter = null; try { if (runner.Load(package)) { testConverter = new TestConverter(testLog, sourceAssembly); int cases = ProcessTestCases(runner.Test, discoverySink, testConverter); testLog.SendDebugMessage(string.Format("Discovered {0} test cases", cases)); } else { testLog.NUnitLoadError(sourceAssembly); } } catch (System.BadImageFormatException) { // we skip the native c++ binaries that we don't support. testLog.AssemblyNotSupportedWarning(sourceAssembly); } catch (System.IO.FileNotFoundException ex) { // Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here testLog.DependentAssemblyNotFoundWarning(ex.FileName, sourceAssembly); } catch (System.Exception ex) { testLog.SendErrorMessage("Exception thrown discovering tests in " + sourceAssembly, ex); } finally { if (testConverter != null) testConverter.Dispose(); testConverter = null; runner.Unload(); } } Info("discovering test", "finished"); }
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { foreach(var source in sources) { var xDomainSink = new TestAdapterLogger(logger, source); xDomainSink.OnBeginTest += (_, e) => discoverySink.SendTestCase(e.TestCase); CrossDomainConeRunner.WithProxyInDomain<ConeTestAdapterProxy, int>(string.Empty, new [] { source, }, proxy => proxy.DiscoverTests(source, xDomainSink) ); } }
public void GetBoostTests(IDictionary<string, ProjectInfo> solutionInfo, ITestCaseDiscoverySink discoverySink) { if (solutionInfo != null) { foreach (KeyValuePair<string, ProjectInfo> info in solutionInfo) { string source = info.Key; ProjectInfo projectInfo = info.Value; foreach(var sourceFile in projectInfo.CppSourceFiles) { try { using (var sr = new StreamReader(sourceFile)) { try { var cppSourceFile = new CppSourceFile() { FileName = sourceFile, SourceCode = sr.ReadToEnd() }; /* * it is important that the pre-processor defines at project level are not modified * because every source file in the project has to have the same starting point. */ ApplySourceFilter(cppSourceFile, new Defines(projectInfo.DefinesHandler)); //call to cpy ctor DiscoverBoostTests(cppSourceFile, source, discoverySink); } catch (Exception ex) { Logger.Error( "Exception raised while discovering tests from \"{0}\" of project \"{1}\", ({2})", sourceFile, projectInfo.ProjectExe, ex.Message); Logger.Error(ex.StackTrace); } } } catch { Logger.Error("Unable to open file \"{0}\" of project \"{1}\".", sourceFile, projectInfo.ProjectExe); } } } } else { Logger.Error("the solutionInfo object was found to be null whilst"); } }
/// <summary> /// ITestDiscover, Given a list of test sources this method pulls out the test cases /// </summary> /// <param name="sources">List of test sources passed from client (Client can be VS or command line)</param> /// <param name="discoveryContext">Context and runSettings for current run. Discoverer pulls out the tests based on current context</param> /// <param name="logger">Used to relay messages to registered loggers</param> /// <param name="discoverySink">Callback used to notify client upon discovery of test cases</param> public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { ValidateArg.NotNull(sources, "sources"); ValidateArg.NotNull(discoverySink, "discoverySink"); ValidateArg.NotNull(logger, "logger"); using (var buildEngine = new MSBuild.ProjectCollection()) { try { // Load all the test containers passed in (.njsproj msbuild files) foreach (string source in sources) { buildEngine.LoadProject(source); } foreach (var proj in buildEngine.LoadedProjects) { var projectHome = Path.GetFullPath(Path.Combine(proj.DirectoryPath, ".")); Dictionary<string, List<TestFileEntry>> testItems = new Dictionary<string,List<TestFileEntry>>(StringComparer.OrdinalIgnoreCase); // Provide all files to the test analyzer foreach (var item in proj.Items.Where(item => item.ItemType == "Compile" || item.ItemType == "TypeScriptCompile")) { //Check to see if this is a TestCase string value = item.GetMetadataValue("TestFramework"); if (!TestContainerDiscoverer.IsValidTestFramework(value)) { continue; } string fileAbsolutePath = CommonUtils.GetAbsoluteFilePath(projectHome, item.EvaluatedInclude); bool typeScriptTest = TypeScript.TypeScriptHelpers.IsTypeScriptFile(fileAbsolutePath); if(typeScriptTest){ fileAbsolutePath = TypeScript.TypeScriptHelpers.GetTypeScriptBackedJavaScriptFile(proj,fileAbsolutePath); } else if (!Path.GetExtension(fileAbsolutePath).Equals(".js", StringComparison.OrdinalIgnoreCase)) { continue; } List<TestFileEntry> fileList; if (!testItems.TryGetValue(value, out fileList)){ fileList = new List<TestFileEntry>(); testItems.Add(value, fileList); } fileList.Add(new TestFileEntry(fileAbsolutePath,typeScriptTest)); } //Debug.Fail("Before Discover"); DiscoverTests(testItems, proj, discoverySink, logger); } } catch (Exception ex) { logger.SendMessage(TestMessageLevel.Error, ex.Message); throw; } finally { // Disposing buildEngine does not clear the document cache in // VS 2013, so manually unload all projects before disposing. buildEngine.UnloadAllProjects(); } } }
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, ITestCaseDiscoverySink discoverySink) { Code.Require(sources, "sources"); Code.Require(discoverySink, "discoverySink"); if (this.Settings.DiscoveryMethodType == DiscoveryMethodType.DiscoveryListContent) { // Delegate to ListContentDiscoverer ListContentDiscoverer discoverer = new ListContentDiscoverer(new ExternalBoostTestRunnerFactory(), VSProvider); discoverer.DiscoverTests(sources, discoveryContext, discoverySink); } }
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { Guard.ArgumentNotNull("sources", sources); Guard.ArgumentNotNull("logger", logger); Guard.ArgumentNotNull("discoverySink", discoverySink); try { RemotingUtility.CleanUpRegisteredChannels(); List<SourceSink> sourceSinks = new List<SourceSink>(); using (AssemblyHelper.SubscribeResolve()) try { foreach (string source in sources) { try { if (cancelled) break; if (IsXunitTestAssembly(source)) { var framework = new Xunit2(SourceInformationProvider, source, configFileName: null, shadowCopy: true); var sink = new VsDiscoveryVisitor(source, framework, logger, discoverySink, () => cancelled); sourceSinks.Add(new SourceSink { Framework = framework, Source = source, Sink = sink }); framework.Find(includeSourceInformation: true, messageSink: sink); } } catch (Exception e) { logger.SendMessage(TestMessageLevel.Error, String.Format("xUnit.net: Exception discovering tests from {0}: {1}", source, e)); } } foreach (var sourceSink in sourceSinks) sourceSink.Sink.Finished.WaitOne(); } finally { foreach (var sourceSink in sourceSinks) { sourceSink.Sink.Dispose(); sourceSink.Framework.Dispose(); } } } catch { } }
private void DiscoverTests(Dictionary <string, HashSet <TestFileEntry> > testItems, MSBuild.Project proj, ITestCaseDiscoverySink discoverySink, IMessageLogger logger) { var result = new List <NodejsTestInfo>(); var projectHome = Path.GetFullPath(Path.Combine(proj.DirectoryPath, ".")); var projSource = proj.FullPath; var nodeExePath = Nodejs.GetAbsoluteNodeExePath( projectHome, proj.GetPropertyValue(NodeProjectProperty.NodeExePath)); if (!File.Exists(nodeExePath)) { logger.SendMessage(TestMessageLevel.Error, "Node.exe was not found. Please install Node.js before running tests."); return; } var testCount = 0; foreach (var testFx in testItems.Keys) { var testFramework = FrameworkDiscoverer.Instance.Get(testFx); if (testFramework == null) { logger.SendMessage(TestMessageLevel.Warning, $"Ignoring unsupported test framework '{testFx}'."); continue; } var fileList = testItems[testFx]; var files = string.Join(";", fileList.Select(p => p.FullPath)); logger.SendMessage(TestMessageLevel.Informational, string.Format(CultureInfo.CurrentCulture, "Processing: {0}", files)); var discoveredTestCases = testFramework.FindTests(fileList.Select(p => p.FullPath), nodeExePath, logger, projectRoot: projectHome); testCount += discoveredTestCases.Count(); foreach (var discoveredTest in discoveredTestCases) { var qualifiedName = discoveredTest.FullyQualifiedName; const string indent = " "; logger.SendMessage(TestMessageLevel.Informational, $"{indent}Creating TestCase:{qualifiedName}"); //figure out the test source info such as line number var filePath = discoveredTest.TestPath; var entry = fileList.First(p => StringComparer.OrdinalIgnoreCase.Equals(p.FullPath, filePath)); FunctionInformation fi = null; if (entry.IsTypeScriptTest) { fi = SourceMapper.MaybeMap(new FunctionInformation(string.Empty, discoveredTest.TestName, discoveredTest.SourceLine, entry.FullPath)); } var testcase = new TestCase(qualifiedName, NodejsConstants.ExecutorUri, projSource) { CodeFilePath = fi?.Filename ?? filePath, LineNumber = fi?.LineNumber ?? discoveredTest.SourceLine, DisplayName = discoveredTest.TestName }; testcase.SetPropertyValue(JavaScriptTestCaseProperties.TestFramework, testFx); testcase.SetPropertyValue(JavaScriptTestCaseProperties.NodeExePath, nodeExePath); testcase.SetPropertyValue(JavaScriptTestCaseProperties.ProjectRootDir, projectHome); testcase.SetPropertyValue(JavaScriptTestCaseProperties.WorkingDir, projectHome); testcase.SetPropertyValue(JavaScriptTestCaseProperties.TestFile, filePath); discoverySink.SendTestCase(testcase); } logger.SendMessage(TestMessageLevel.Informational, string.Format(CultureInfo.CurrentCulture, "Processing finished for framework '{0}'.", testFx)); } }
internal void SendTestCases(string source, IEnumerable <UnitTestElement> testElements, ITestCaseDiscoverySink discoverySink) { var shouldCollectSourceInformation = MSTestSettings.RunConfigurationSettings.CollectSourceInformation; var navigationSessions = new Dictionary <string, object>(); try { if (shouldCollectSourceInformation) { navigationSessions.Add(source, PlatformServiceProvider.Instance.FileOperations.CreateNavigationSession(source)); } foreach (var testElement in testElements) { object testNavigationSession; var testCase = testElement.ToTestCase(); if (shouldCollectSourceInformation) { string testSource = testElement.TestMethod.DeclaringAssemblyName ?? source; if (!navigationSessions.TryGetValue(testSource, out testNavigationSession)) { testNavigationSession = PlatformServiceProvider.Instance.FileOperations.CreateNavigationSession(testSource); navigationSessions.Add(testSource, testNavigationSession); } if (testNavigationSession != null) { var className = testElement.TestMethod.DeclaringClassFullName ?? testElement.TestMethod.FullClassName; var methodName = testElement.TestMethod.Name; // If it is async test method use compiler generated type and method name for navigation data. if (!string.IsNullOrEmpty(testElement.AsyncTypeName)) { className = testElement.AsyncTypeName; // compiler generated method name is "MoveNext". methodName = "MoveNext"; } int minLineNumber; string fileName; PlatformServiceProvider.Instance.FileOperations.GetNavigationData( testNavigationSession, className, methodName, out minLineNumber, out fileName); if (!string.IsNullOrEmpty(fileName)) { testCase.LineNumber = minLineNumber; testCase.CodeFilePath = fileName; } } } discoverySink.SendTestCase(testCase); } } finally { foreach (object navigationSession in navigationSessions.Values) { PlatformServiceProvider.Instance.FileOperations.DisposeNavigationSession(navigationSession); } } }
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { throw new NotImplementedException(); }
public VsTestFrameworkReporter(ITestCaseDiscoverySink sink, ILogger logger) : this(sink, null, false, logger) { }
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { // indicate start of discovery logger.SendMessage(TestMessageLevel.Informational, "Machine Specifications Visual Studio Test Adapter - Discovering Specifications."); int discoveredSpecCount = 0; int sourcesWithSpecs = 0; Settings settings = GetSettings(discoveryContext); foreach (string assemblyPath in sources.Distinct()) { try { #if !NETSTANDARD if (!File.Exists(Path.Combine(Path.GetDirectoryName(Path.GetFullPath(assemblyPath)), "Machine.Specifications.dll"))) { continue; } #endif sourcesWithSpecs++; logger.SendMessage(TestMessageLevel.Informational, string.Format("Machine Specifications Visual Studio Test Adapter - Discovering...looking in {0}", assemblyPath)); List <TestCase> specs = discoverer.DiscoverSpecs(assemblyPath) .Select(spec => SpecTestHelper.GetVSTestCaseFromMSpecTestCase(assemblyPath, spec, settings.DisableFullTestNameInIDE, MSpecTestAdapter.uri)) .ToList(); foreach (TestCase discoveredTest in specs) { discoveredSpecCount++; if (discoverySink != null) { discoverySink.SendTestCase(discoveredTest); } } } catch (Exception discoverException) { logger.SendMessage(TestMessageLevel.Error, string.Format("Machine Specifications Visual Studio Test Adapter - Error while discovering specifications in assembly {0} - {1}", assemblyPath, discoverException.Message)); } } // indicate that we are finished discovering logger.SendMessage(TestMessageLevel.Informational, string.Format("Machine Specifications Visual Studio Test Adapter - Discovery Complete - {0} specifications in {2} of {1} assemblies scanned.", discoveredSpecCount, sources.Count(), sourcesWithSpecs)); }
/// <summary> /// ITestDiscover, Given a list of test sources this method pulls out the test cases /// </summary> /// <param name="sources">List of test sources passed from client (Client can be VS or command line)</param> /// <param name="logger">Used to relay messages to registered loggers</param> /// <param name="discoverySink">Callback used to notify client upon discovery of test cases</param> private void DiscoverTestsCore(IEnumerable <string> sources, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { ValidateArg.NotNull(sources, nameof(sources)); ValidateArg.NotNull(discoverySink, nameof(discoverySink)); ValidateArg.NotNull(logger, nameof(logger)); var env = new Dictionary <string, string>(); var root = Environment.GetEnvironmentVariable("VSINSTALLDIR"); #if DEBUG logger.SendMessage(TestMessageLevel.Informational, $"VSINSTALLDIR: {root}"); #endif if (!string.IsNullOrEmpty(root)) { env["VsInstallRoot"] = root; env["MSBuildExtensionsPath32"] = Path.Combine(root, "MSBuild"); } using (var buildEngine = new MSBuild.ProjectCollection(env)) { try { // Load all the test containers passed in (.njsproj msbuild files) foreach (var source in sources) { var cleanPath = source.Trim('\'', '"'); buildEngine.LoadProject(cleanPath); } FrameworkDiscoverer frameworkDiscoverer = null; foreach (var proj in buildEngine.LoadedProjects) { var projectHome = Path.GetFullPath(Path.Combine(proj.DirectoryPath, ".")); var testItems = new Dictionary <string, HashSet <string> >(StringComparer.OrdinalIgnoreCase); var testRoot = proj.GetProperty(NodeProjectProperty.TestRoot)?.EvaluatedValue; var testFramework = proj.GetProperty(NodeProjectProperty.TestFramework)?.EvaluatedValue; if (!string.IsNullOrEmpty(testRoot) && string.IsNullOrEmpty(testFramework)) { logger.SendMessage(TestMessageLevel.Warning, $"TestRoot specified for '{Path.GetFileName(proj.FullPath)}' but no TestFramework."); } // Provide all files to the test analyzer foreach (var item in proj.Items) { string testFrameworkName; string fileAbsolutePath; if (!string.IsNullOrEmpty(testRoot) && !string.IsNullOrEmpty(testFramework)) { testFrameworkName = testFramework; var testRootPath = Path.GetFullPath(Path.Combine(proj.DirectoryPath, testRoot)); try { fileAbsolutePath = CommonUtils.GetAbsoluteFilePath(projectHome, item.EvaluatedInclude); } catch (ArgumentException) { // .Net core projects include invalid paths, ignore them and continue checking the items. continue; } if (!fileAbsolutePath.StartsWith(testRootPath, StringComparison.OrdinalIgnoreCase)) { continue; } } else { //Check to see if this is a TestCase testFrameworkName = item.GetMetadataValue("TestFramework"); if (!TestFramework.IsValidTestFramework(testFrameworkName)) { continue; } fileAbsolutePath = CommonUtils.GetAbsoluteFilePath(projectHome, item.EvaluatedInclude); } var typeScriptTest = TypeScript.TypeScriptHelpers.IsTypeScriptFile(fileAbsolutePath); if (typeScriptTest) { fileAbsolutePath = TypeScript.TypeScriptHelpers.GetTypeScriptBackedJavaScriptFile(proj, fileAbsolutePath); } else if (!StringComparer.OrdinalIgnoreCase.Equals(Path.GetExtension(fileAbsolutePath), ".js")) { continue; } if (!testItems.TryGetValue(testFrameworkName, out var fileList)) { fileList = new HashSet <string>(StringComparer.OrdinalIgnoreCase); testItems.Add(testFrameworkName, fileList); } fileList.Add(fileAbsolutePath); } if (testItems.Any()) { frameworkDiscoverer = frameworkDiscoverer ?? new FrameworkDiscoverer(); var nodeExePath = Nodejs.GetAbsoluteNodeExePath(projectHome, proj.GetPropertyValue(NodeProjectProperty.NodeExePath)); if (string.IsNullOrEmpty(nodeExePath)) { // if nothing specified in the project fallback to environment nodeExePath = Nodejs.GetPathToNodeExecutableFromEnvironment(); } var spaRoot = proj.GetProperty("SpaRoot")?.EvaluatedValue; var workDir = Path.GetDirectoryName(proj.FullPath); if (!String.IsNullOrEmpty(spaRoot)) { workDir = Path.Combine(workDir, spaRoot); } this.DiscoverTests(testItems, frameworkDiscoverer, discoverySink, logger, nodeExePath, proj.FullPath, workDir); } } } catch (Exception ex) { logger.SendMessage(TestMessageLevel.Error, ex.Message); throw; } finally { // Disposing buildEngine does not clear the document cache in // VS 2013, so manually unload all projects before disposing. buildEngine.UnloadAllProjects(); } } }
public virtual void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { DiscoveryContext = discoveryContext; MessageLogger = logger; DiscoverySink = discoverySink; var testCase = new TestCase("A.C.M", new Uri("executor://dllexecutor"), "A"); discoverySink.SendTestCase(testCase); }
/// <summary> /// ITestDiscover, Given a list of test sources this method pulls out the test cases /// </summary> /// <param name="sources">List of test sources passed from client (Client can be VS or command line)</param> /// <param name="discoveryContext">Context and runSettings for current run. Discoverer pulls out the tests based on current context</param> /// <param name="logger">Used to relay messages to registered loggers</param> /// <param name="discoverySink">Callback used to notify client upon discovery of test cases</param> public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { ValidateArg.NotNull(sources, "sources"); ValidateArg.NotNull(discoverySink, "discoverySink"); ValidateArg.NotNull(logger, "logger"); var env = new Dictionary <string, string>(); #if DEV15 var root = Environment.GetEnvironmentVariable(NodejsConstants.NodeToolsVsInstallRootEnvironmentVariable); if (!string.IsNullOrEmpty(root)) { env["VsInstallRoot"] = root; env["MSBuildExtensionsPath32"] = Path.Combine(root, "MSBuild"); } #endif using (var buildEngine = new MSBuild.ProjectCollection(env)) { try { // Load all the test containers passed in (.njsproj msbuild files) foreach (string source in sources) { buildEngine.LoadProject(source); } foreach (var proj in buildEngine.LoadedProjects) { var projectHome = Path.GetFullPath(Path.Combine(proj.DirectoryPath, ".")); Dictionary <string, List <TestFileEntry> > testItems = new Dictionary <string, List <TestFileEntry> >(StringComparer.OrdinalIgnoreCase); // Provide all files to the test analyzer foreach (var item in proj.Items.Where(item => item.ItemType == "Compile" || item.ItemType == "TypeScriptCompile")) { //Check to see if this is a TestCase string value = item.GetMetadataValue("TestFramework"); if (!TestContainerDiscoverer.IsValidTestFramework(value)) { continue; } string fileAbsolutePath = CommonUtils.GetAbsoluteFilePath(projectHome, item.EvaluatedInclude); bool typeScriptTest = TypeScript.TypeScriptHelpers.IsTypeScriptFile(fileAbsolutePath); if (typeScriptTest) { fileAbsolutePath = TypeScript.TypeScriptHelpers.GetTypeScriptBackedJavaScriptFile(proj, fileAbsolutePath); } else if (!Path.GetExtension(fileAbsolutePath).Equals(".js", StringComparison.OrdinalIgnoreCase)) { continue; } List <TestFileEntry> fileList; if (!testItems.TryGetValue(value, out fileList)) { fileList = new List <TestFileEntry>(); testItems.Add(value, fileList); } fileList.Add(new TestFileEntry(fileAbsolutePath, typeScriptTest)); } //Debug.Fail("Before Discover"); DiscoverTests(testItems, proj, discoverySink, logger); } } catch (Exception ex) { logger.SendMessage(TestMessageLevel.Error, ex.Message); throw; } finally { // Disposing buildEngine does not clear the document cache in // VS 2013, so manually unload all projects before disposing. buildEngine.UnloadAllProjects(); } } }
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { // indicate start of discovery logger.SendMessage(TestMessageLevel.Informational, Strings.DISCOVERER_STARTING); int discoveredSpecCount = 0; int sourcesWithSpecs = 0; Settings settings = GetSettings(discoveryContext); foreach (string assemblyPath in sources.Distinct()) { try { if (!File.Exists(Path.Combine(Path.GetDirectoryName(Path.GetFullPath(assemblyPath)), "Machine.Specifications.dll"))) { continue; } sourcesWithSpecs++; logger.SendMessage(TestMessageLevel.Informational, string.Format(Strings.DISCOVERER_LOOKINGIN, assemblyPath)); List <TestCase> specs = discoverer.DiscoverSpecs(assemblyPath) .Select(spec => SpecTestHelper.GetVSTestCaseFromMSpecTestCase(assemblyPath, spec, settings.DisableFullTestNameInIDE, MSpecTestAdapter.uri, CreateTrait)) .ToList(); foreach (TestCase discoveredTest in specs) { discoveredSpecCount++; if (discoverySink != null) { discoverySink.SendTestCase(discoveredTest); } } } catch (Exception discoverException) { logger.SendMessage(TestMessageLevel.Error, string.Format(Strings.DISCOVERER_ERROR, assemblyPath, discoverException.Message)); } } // indicate that we are finished discovering logger.SendMessage(TestMessageLevel.Informational, string.Format(Strings.DISCOVERER_COMPLETE, discoveredSpecCount, sources.Count(), sourcesWithSpecs)); }
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { var configuration = new Mocha.BddConfiguration(); //TODO: unhardcode var tests = GetTests(configuration, sources); foreach (var test in tests) { discoverySink.SendTestCase(test); } }
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger, ITestCaseDiscoverySink discoverySink) { #if LAUNCHDEBUGGER if (!Debugger.IsAttached) { Debugger.Launch(); } #endif Initialize(discoveryContext, messageLogger); TestLog.Info($"NUnit Adapter {AdapterVersion}: Test discovery starting"); // Ensure any channels registered by other adapters are unregistered CleanUpRegisteredChannels(); if (Settings.InProcDataCollectorsAvailable && sources.Count() > 1) { TestLog.Error("Unexpected to discover tests in multiple assemblies when InProcDataCollectors specified in run configuration."); Unload(); return; } foreach (string sourceAssembly in sources) { string sourceAssemblyPath = Path.IsPathRooted(sourceAssembly) ? sourceAssembly : Path.Combine(Directory.GetCurrentDirectory(), sourceAssembly); TestLog.Debug("Processing " + sourceAssembly); if (Settings.DumpXmlTestDiscovery) { dumpXml = new DumpXml(sourceAssemblyPath); } try { var package = CreateTestPackage(sourceAssemblyPath, null); NUnitEngineAdapter.CreateRunner(package); var results = NUnitEngineAdapter.Explore(); dumpXml?.AddString(results.AsString()); if (results.IsRunnable) { int cases; using (var testConverter = new TestConverterForXml(TestLog, sourceAssemblyPath, Settings)) { var timing = new TimingLogger(Settings, TestLog); cases = ProcessTestCases(results, discoverySink, testConverter); timing.LogTime("Discovery/Processing/Converting:"); } TestLog.Debug($"Discovered {cases} test cases"); // Only save if seed is not specified in runsettings // This allows workaround in case there is no valid // location in which the seed may be saved. if (cases > 0 && !Settings.RandomSeedSpecified) { Settings.SaveRandomSeed(Path.GetDirectoryName(sourceAssemblyPath)); } } else { if (results.HasNoNUnitTests) { if (Settings.Verbosity > 0) { TestLog.Info("Assembly contains no NUnit 3.0 tests: " + sourceAssembly); } } else { TestLog.Info("NUnit failed to load " + sourceAssembly); } } } catch (NUnitEngineException e) { if (e.InnerException is BadImageFormatException) { // we skip the native c++ binaries that we don't support. TestLog.Warning("Assembly not supported: " + sourceAssembly); } else { TestLog.Warning("Exception thrown discovering tests in " + sourceAssembly, e); } } catch (BadImageFormatException) { // we skip the native c++ binaries that we don't support. TestLog.Warning("Assembly not supported: " + sourceAssembly); } catch (FileNotFoundException ex) { // Either the NUnit framework was not referenced by the test assembly // or some other error occurred. Not a problem if not an NUnit assembly. TestLog.Warning("Dependent Assembly " + ex.FileName + " of " + sourceAssembly + " not found. Can be ignored if not an NUnit project."); } catch (FileLoadException ex) { // Attempts to load an invalid assembly, or an assembly with missing dependencies TestLog.Warning("Assembly " + ex.FileName + " loaded through " + sourceAssembly + " failed. Assembly is ignored. Correct deployment of dependencies if this is an error."); } catch (TypeLoadException ex) { if (ex.TypeName == "NUnit.Framework.Api.FrameworkController") { TestLog.Warning(" Skipping NUnit 2.x test assembly"); } else { TestLog.Warning("Exception thrown discovering tests in " + sourceAssembly, ex); } } catch (Exception ex) { TestLog.Warning("Exception thrown discovering tests in " + sourceAssembly, ex); } finally { dumpXml?.DumpForDiscovery(); NUnitEngineAdapter?.CloseRunner(); } } TestLog.Info($"NUnit Adapter {AdapterVersion}: Test discovery complete"); Unload(); }
public XmlTestCaseReader(ITestCaseDiscoverySink testCaseSink) { this.testCaseSink = testCaseSink; _testCases = new List <TestCase>(); }
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger, ITestCaseDiscoverySink discoverySink) { TestLog.Initialize(messageLogger); if (RegistryFailure) { TestLog.SendErrorMessage(ErrorMsg); } Info("discovering tests", "started"); // Ensure any channels registered by other adapters are unregistered CleanUpRegisteredChannels(); foreach (string sourceAssembly in sources) { TestLog.SendDebugMessage("Processing " + sourceAssembly); TestRunner runner = new TestDomain(); var package = CreateTestPackage(sourceAssembly); TestConverter testConverter = null; try { if (runner.Load(package)) { testConverter = new TestConverter(TestLog, sourceAssembly); int cases = ProcessTestCases(runner.Test, discoverySink, testConverter); TestLog.SendDebugMessage(string.Format("Discovered {0} test cases", cases)); } else { TestLog.NoNUnit2TestsFoundIn(sourceAssembly); } } catch (BadImageFormatException) { // we skip the native c++ binaries that we don't support. TestLog.AssemblyNotSupportedWarning(sourceAssembly); } catch (FileNotFoundException ex) { // Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here TestLog.DependentAssemblyNotFoundWarning(ex.FileName, sourceAssembly); } catch (FileLoadException ex) { // Attempts to load an invalid assembly, or an assembly with missing dependencies TestLog.LoadingAssemblyFailedWarning(ex.FileName, sourceAssembly); } catch (UnsupportedFrameworkException) { TestLog.UnsupportedFrameworkWarning(sourceAssembly); } catch (Exception ex) { TestLog.SendErrorMessage("Exception thrown discovering tests in " + sourceAssembly, ex); } finally { runner.Unload(); } } Info("discovering test", "finished"); }
private void DiscoverTests(Dictionary <string, HashSet <string> > testItems, FrameworkDiscoverer frameworkDiscoverer, ITestCaseDiscoverySink discoverySink, IMessageLogger logger, string nodeExePath, string projectFullPath, string workDir) { foreach (var testFx in testItems.Keys) { var testFramework = frameworkDiscoverer.GetFramework(testFx); if (testFramework == null) { logger.SendMessage(TestMessageLevel.Warning, $"Ignoring unsupported test framework '{testFx}'."); continue; } var fileList = testItems[testFx]; var discoverWorker = new TestDiscovererWorker(projectFullPath, NodejsConstants.ExecutorUri, nodeExePath, workDir); discoverWorker.DiscoverTests(fileList, testFramework, logger, discoverySink); } }
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { AssemblyResolver.SetupHandler(); this.DiscoverTestsCore(sources, logger, discoverySink); }
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { IsDiscoverTestCalled = true; Sources = sources; DiscoveryContext = discoveryContext; MessageLogger = logger; DiscoverySink = discoverySink; }
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { ChutzpahTracer.TraceInformation("Begin Test Adapter Discover Tests"); var settingsProvider = discoveryContext.RunSettings.GetSettings(AdapterConstants.SettingsName) as ChutzpahAdapterSettingsProvider; var settings = settingsProvider != null ? settingsProvider.Settings : new ChutzpahAdapterSettings(); ChutzpahTracingHelper.Toggle(settings.EnabledTracing); var testOptions = new TestOptions { MaxDegreeOfParallelism = settings.MaxDegreeOfParallelism, ChutzpahSettingsFileEnvironments = new ChutzpahSettingsFileEnvironments(settings.ChutzpahSettingsFileEnvironments) }; IList <TestError> errors; var testCases = testRunner.DiscoverTests(sources, testOptions, out errors); ChutzpahTracer.TraceInformation("Sending discovered tests to test case discovery sink"); foreach (var testCase in testCases) { var vsTestCase = testCase.ToVsTestCase(); discoverySink.SendTestCase(vsTestCase); } foreach (var error in errors) { logger.SendMessage(TestMessageLevel.Error, RunnerCallback.FormatFileErrorMessage(error)); } ChutzpahTracer.TraceInformation("End Test Adapter Discover Tests"); }
public override void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { Sources = sources; IsNativeDiscoverTestCalled = true; base.DiscoverTests(sources, discoveryContext, logger, discoverySink); }
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger, ITestCaseDiscoverySink discoverySink) { #if LAUNCHDEBUGGER if (!Debugger.IsAttached) { Debugger.Launch(); } #endif Initialize(discoveryContext, messageLogger); TestLog.Info(string.Format("NUnit Adapter {0}: Test discovery starting", AdapterVersion)); // Ensure any channels registered by other adapters are unregistered CleanUpRegisteredChannels(); foreach (string sourceAssembly in sources) { TestLog.Debug("Processing " + sourceAssembly); ITestRunner runner = null; runner = GetRunnerFor(sourceAssembly); try { XmlNode loadResult = runner.Load(); // Currently, this will always be the case but it might change if (loadResult.Name == "test-run") { loadResult = loadResult.FirstChild; } if (loadResult.GetAttribute("runstate") == "Runnable") { XmlNode topNode = runner.Explore(TestFilter.Empty); using (var testConverter = new TestConverter(TestLog, sourceAssembly)) { int cases = ProcessTestCases(topNode, discoverySink, testConverter); TestLog.Debug(string.Format("Discovered {0} test cases", cases)); } } else { var msgNode = loadResult.SelectSingleNode("properties/property[@name='_SKIPREASON']"); if (msgNode != null && (new[] { "contains no tests", "Has no TestFixtures" }).Any(msgNode.GetAttribute("value").Contains)) { TestLog.Info("Assembly contains no NUnit 3.0 tests: " + sourceAssembly); } else { TestLog.Info("NUnit failed to load " + sourceAssembly); } } } catch (BadImageFormatException) { // we skip the native c++ binaries that we don't support. TestLog.Warning("Assembly not supported: " + sourceAssembly); } catch (FileNotFoundException ex) { // Either the NUnit framework was not referenced by the test assembly // or some other error occured. Not a problem if not an NUnit assembly. TestLog.Warning("Dependent Assembly " + ex.FileName + " of " + sourceAssembly + " not found. Can be ignored if not a NUnit project."); } catch (FileLoadException ex) { // Attempts to load an invalid assembly, or an assembly with missing dependencies TestLog.Warning("Assembly " + ex.FileName + " loaded through " + sourceAssembly + " failed. Assembly is ignored. Correct deployment of dependencies if this is an error."); } catch (TypeLoadException ex) { if (ex.TypeName == "NUnit.Framework.Api.FrameworkController") { TestLog.Warning(" Skipping NUnit 2.x test assembly"); } else { TestLog.Error("Exception thrown discovering tests in " + sourceAssembly, ex); } } catch (Exception ex) { TestLog.Error("Exception thrown discovering tests in " + sourceAssembly, ex); } finally { if (runner.IsTestRunning) { runner.StopRun(true); } runner.Unload(); runner.Dispose(); } } TestLog.Info(string.Format("NUnit Adapter {0}: Test discovery complete", AdapterVersion)); Unload(); }
private void DiscoverTests(Dictionary <string, List <TestFileEntry> > testItems, MSBuild.Project proj, ITestCaseDiscoverySink discoverySink, IMessageLogger logger) { List <TestFrameworks.NodejsTestInfo> result = new List <TestFrameworks.NodejsTestInfo>(); var projectHome = Path.GetFullPath(Path.Combine(proj.DirectoryPath, ".")); var projSource = ((MSBuild.Project)proj).FullPath; var nodeExePath = Nodejs.GetAbsoluteNodeExePath( projectHome, proj.GetPropertyValue(NodejsConstants.NodeExePath)); if (!File.Exists(nodeExePath)) { logger.SendMessage(TestMessageLevel.Error, String.Format("Node.exe was not found. Please install Node.js before running tests.")); return; } int testCount = 0; foreach (string testFx in testItems.Keys) { TestFrameworks.TestFramework testFramework = GetTestFrameworkObject(testFx); if (testFramework == null) { logger.SendMessage(TestMessageLevel.Warning, String.Format("Ignoring unsupported test framework {0}", testFx)); continue; } List <TestFileEntry> fileList = testItems[testFx]; string files = string.Join(";", fileList.Select(p => p.File)); logger.SendMessage(TestMessageLevel.Informational, String.Format("Processing: {0}", files)); List <TestFrameworks.NodejsTestInfo> discoveredTestCases = testFramework.FindTests(fileList.Select(p => p.File), nodeExePath, logger, projectHome); testCount += discoveredTestCases.Count; foreach (TestFrameworks.NodejsTestInfo discoveredTest in discoveredTestCases) { string qualifiedName = discoveredTest.FullyQualifiedName; logger.SendMessage(TestMessageLevel.Informational, String.Format(" " /*indent*/ + "Creating TestCase:{0}", qualifiedName)); //figure out the test source info such as line number string filePath = discoveredTest.ModulePath; TestFileEntry entry = fileList.Find(p => p.File.Equals(filePath, StringComparison.OrdinalIgnoreCase)); FunctionInformation fi = null; if (entry.IsTypeScriptTest) { fi = SourceMapper.MaybeMap(new FunctionInformation(String.Empty, discoveredTest.TestName, discoveredTest.SourceLine, entry.File)); } discoverySink.SendTestCase( new TestCase(qualifiedName, TestExecutor.ExecutorUri, projSource) { CodeFilePath = (fi != null) ? fi.Filename : filePath, LineNumber = (fi != null && fi.LineNumber.HasValue) ? fi.LineNumber.Value : discoveredTest.SourceLine, DisplayName = discoveredTest.TestName }); } logger.SendMessage(TestMessageLevel.Informational, string.Format("Processing finished for framework of {0}", testFx)); } if (testCount == 0) { logger.SendMessage(TestMessageLevel.Warning, String.Format("Discovered 0 testcases.")); } }
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, ITestCaseDiscoverySink discoverySink) { Code.Require(sources, "sources"); Code.Require(discoverySink, "discoverySink"); // Populate loop-invariant attributes and settings BoostTestAdapterSettings settings = BoostTestAdapterSettingsProvider.GetSettings(discoveryContext); BoostTestRunnerSettings runnerSettings = new BoostTestRunnerSettings() { Timeout = settings.DiscoveryTimeoutMilliseconds }; BoostTestRunnerCommandLineArgs args = new BoostTestRunnerCommandLineArgs() { ListContent = ListContentFormat.DOT }; foreach (var source in sources) { try { using (new Utility.TimedScope("Extract settings from Visual Studio for \"{0}\"", source)) { var vs = _vsProvider?.Instance; if (vs != null) { Logger.Debug("Connected to Visual Studio {0} instance", vs.Version); } args.SetWorkingEnvironment(source, settings, vs); } } catch (ROTException ex) { Logger.Exception(ex, "Could not retrieve WorkingDirectory from Visual Studio Configuration"); } catch (COMException ex) { Logger.Exception(ex, "Could not retrieve WorkingDirectory from Visual Studio Configuration"); } try { IBoostTestRunner runner = _factory.GetRunner(source, settings.TestRunnerFactoryOptions); using (TemporaryFile output = new TemporaryFile(TestPathGenerator.Generate(source, ".list.content.gv"))) { // --list_content output is redirected to standard error args.StandardErrorFile = output.Path; Logger.Debug("list_content file: {0}", args.StandardErrorFile); int resultCode = EXIT_SUCCESS; using (new Utility.TimedScope("Execute \"--list_content=DOT\" for \"{0}\"", source)) { using (var context = new DefaultProcessExecutionContext()) { resultCode = runner.Execute(args, runnerSettings, context); } } // Skip sources for which the --list_content file is not available if (!File.Exists(args.StandardErrorFile)) { Logger.Error("--list_content=DOT output for {0} is not available. Skipping.", source); continue; } // If the executable failed to exit with an EXIT_SUCCESS code, skip source and notify user accordingly if (resultCode != EXIT_SUCCESS) { Logger.Error("--list_content=DOT for {0} failed with exit code {1}. Skipping.", source, resultCode); continue; } // Parse --list_content=DOT output Stream stream = null; try { stream = File.OpenRead(args.StandardErrorFile); using (var reader = new StreamReader(stream, System.Text.Encoding.Default)) { stream = null; TestFrameworkDOTDeserialiser deserialiser = new TestFrameworkDOTDeserialiser(source); TestFramework framework = deserialiser.Deserialise(reader); if ((framework != null) && (framework.MasterTestSuite != null)) { framework.MasterTestSuite.Apply(new VSDiscoveryVisitor(source, GetVersion(runner), discoverySink)); } } } finally { if (stream != null) { stream.Dispose(); } } } } catch (Exception ex) { Logger.Exception(ex, "Exception caught while discovering tests for {0} ({1} - {2})", source, ex.Message, ex.HResult); } } }
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) { if (sources == null) { throw new ArgumentNullException(nameof(sources)); } if (discoverySink == null) { throw new ArgumentNullException(nameof(discoverySink)); } var settings = discoveryContext.RunSettings; DiscoverTests(sources, logger, discoverySink, settings); }
static void DiscoverTests(IEnumerable <string> sources, IMessageLogger logger, ITestCaseDiscoverySink discoverySink, IRunSettings settings) { var sourcesSet = new HashSet <string>(sources, StringComparer.OrdinalIgnoreCase); var executorUri = new Uri(PythonConstants.TestExecutorUriString); // Test list is sent to us via our run settings which we use to smuggle the // data we have in our analysis process. var doc = Read(settings.SettingsXml); foreach (var t in TestReader.ReadTests(doc, sourcesSet, m => { logger?.SendMessage(TestMessageLevel.Warning, m); })) { var tc = new TestCase(t.FullyQualifiedName, executorUri, t.SourceFile) { DisplayName = t.DisplayName, LineNumber = t.LineNo, CodeFilePath = t.FileName }; discoverySink.SendTestCase(tc); } }