Example #1
0
        public void RunAssembly(IFrameworkHandle testLog)
        {
            try
            {
#if LAUNCHDEBUGGER
                System.Diagnostics.Debugger.Launch();
#endif
                if (TryLoadAssembly())
                {
                    using (NUnitEventListener listener = new NUnitEventListener(testLog, TestConverter))
                    {
                        try
                        {
                            runner.Run(listener, NUnitFilter, true, LoggingThreshold.Off);
                        }
                        catch (NullReferenceException)
                        {
                            // this happens during the run when CancelRun is called.
                            logger.SendDebugMessage("Nullref caught");
                        }
                        finally
                        {
                            runner.Unload();
                        }
                    }
                }
                else
                {
                    logger.NoNUnit2TestsFoundIn(assemblyName);
                }
            }
            catch (BadImageFormatException)
            {
                // we skip the native c++ binaries that we don't support.
                logger.AssemblyNotSupportedWarning(assemblyName);
            }
            catch (System.IO.FileNotFoundException ex)
            {
                // Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
                logger.DependentAssemblyNotFoundWarning(ex.FileName, assemblyName);
            }
            catch (UnsupportedFrameworkException ex)
            {
                //The UnsupportedFrameworkException is thrown by nunit, if the assembly references the NUnit v3 framework.
                logger.SendInformationalMessage("Attempt to load assembly with unsupported test framework in " + assemblyName);
            }
            catch (Exception ex)
            {
                logger.SendErrorMessage("Exception thrown executing tests in " + assemblyName, ex);
            }
        }
Example #2
0
        public virtual bool Run(TestFilter filter, DiscoveryConverter discovery, NUnit3TestExecutor nUnit3TestExecutor)
        {
            filter = CheckFilterInCurrentMode(filter, discovery);
            nUnit3TestExecutor.Dump?.StartExecution(filter, "(At Execution)");
            var converter = CreateConverter(discovery);

            using var listener = new NUnitEventListener(converter, nUnit3TestExecutor);
            try
            {
                var results = NUnitEngineAdapter.Run(listener, filter);
                NUnitEngineAdapter.GenerateTestOutput(results, discovery.AssemblyPath, TestOutputXmlFolder);
            }
            catch (NullReferenceException)
            {
                // this happens during the run when CancelRun is called.
                TestLog.Debug("   Null ref caught");
            }

            return(true);
        }
        private void RunAssembly(string assemblyName, IFrameworkHandle frameworkHandle)
        {
            #if LAUNCHDEBUGGER
            if (!Debugger.IsAttached)
                Debugger.Launch();
            #endif
            _testRunner = GetRunnerFor(assemblyName);

            try
            {
                var loadResult = _testRunner.Explore(TestFilter.Empty);

                if (loadResult.Name == "test-run")
                    loadResult = loadResult.FirstChild;

                if (loadResult.GetAttribute("runstate") == "Runnable")
                {
                    TestLog.SendInformationalMessage(string.Format("Loading tests from {0}", assemblyName));

                    var nunitTestCases = loadResult.SelectNodes("//test-case");

                    using (var testConverter = new TestConverter(TestLog, assemblyName))
                    {
                        var loadedTestCases = new List<TestCase>();

                        // As a side effect of calling TestConverter.ConvertTestCase,
                        // the converter's cache of all test cases is populated as well.
                        // All future calls to convert a test case may now use the cache.
                        foreach (XmlNode testNode in nunitTestCases)
                            loadedTestCases.Add(testConverter.ConvertTestCase(testNode));

                        // If we have a TFS Filter, convert it to an nunit filter
                        if (_tfsFilter != null && _tfsFilter.HasTfsFilterValue)
                        {
                            var filteredTestCases = _tfsFilter.CheckFilter(loadedTestCases);
                            var testCases = filteredTestCases as TestCase[] ?? filteredTestCases.ToArray();
                            TestLog.SendInformationalMessage(string.Format("TFS Filter detected: LoadedTestCases {0}, Filterered Test Cases {1}", loadedTestCases.Count, testCases.Count()));
                            _nunitFilter = MakeTestFilter(testCases);
                        }

                        using (var listener = new NUnitEventListener(frameworkHandle, testConverter))
                        {
                            try
                            {
                                _testRunner.Run(listener, _nunitFilter);
                            }
                            catch (NullReferenceException)
                            {
                                // this happens during the run when CancelRun is called.
                                TestLog.SendDebugMessage("Nullref caught");
                            }
                        }
                    }
                }
                else
                    TestLog.NUnitLoadError(assemblyName);
            }
            catch (BadImageFormatException)
            {
                // we skip the native c++ binaries that we don't support.
                TestLog.AssemblyNotSupportedWarning(assemblyName);
            }
            catch (System.IO.FileNotFoundException ex)
            {
                // Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
                TestLog.DependentAssemblyNotFoundWarning(ex.FileName, assemblyName);
            }
            catch (Exception ex)
            {
                if (ex is TargetInvocationException)
                    ex = ex.InnerException;
                TestLog.SendErrorMessage("Exception thrown executing tests in " + assemblyName, ex);
            }
            _testRunner.Dispose();
        }
Example #4
0
        private void RunAssembly(string assemblyPath, TestFilter filter)
        {
#if LAUNCHDEBUGGER
            if (!Debugger.IsAttached)
            {
                Debugger.Launch();
            }
#endif

            var actionText    = Debugger.IsAttached ? "Debugging " : "Running ";
            var selectionText = filter == null || filter == TestFilter.Empty ? "all" : "selected";
            TestLog.Info(actionText + selectionText + " tests in " + assemblyPath);

            // No need to restore if the seed was in runsettings file
            if (!Settings.RandomSeedSpecified)
            {
                Settings.RestoreRandomSeed(Path.GetDirectoryName(assemblyPath));
            }
            DumpXml dumpXml = null;
            if (Settings.DumpXmlTestResults)
            {
                dumpXml = new Dump.DumpXml(assemblyPath);
            }

            try
            {
                _activeRunner = GetRunnerFor(assemblyPath);

                var loadResult = _activeRunner.Explore(TestFilter.Empty);
#if !NETCOREAPP1_0
                dumpXml?.AddString(loadResult.AsString());
#endif
                if (loadResult.Name == "test-run")
                {
                    loadResult = loadResult.FirstChild;
                }

                if (loadResult.GetAttribute("runstate") == "Runnable")
                {
                    var nunitTestCases = loadResult.SelectNodes("//test-case");

                    using (var testConverter = new TestConverter(TestLog, assemblyPath, Settings.CollectSourceInformation))
                    {
                        var loadedTestCases = new List <TestCase>();

                        // As a side effect of calling TestConverter.ConvertTestCase,
                        // the converter's cache of all test cases is populated as well.
                        // All future calls to convert a test case may now use the cache.
                        foreach (XmlNode testNode in nunitTestCases)
                        {
                            loadedTestCases.Add(testConverter.ConvertTestCase(testNode));
                        }


                        TestLog.Info($"NUnit3TestExecutor converted {loadedTestCases.Count} of {nunitTestCases.Count} NUnit test cases");

                        // If we have a TFS Filter, convert it to an nunit filter
                        if (TfsFilter != null && !TfsFilter.IsEmpty)
                        {
                            // NOTE This overwrites filter used in call
                            var filterBuilder = CreateTestFilterBuilder();
                            filter = filterBuilder.ConvertTfsFilterToNUnitFilter(TfsFilter, loadedTestCases);
                        }

                        if (filter == NUnitTestFilterBuilder.NoTestsFound)
                        {
                            TestLog.Info("Skipping assembly - no matching test cases found");
                            return;
                        }

                        using (var listener = new NUnitEventListener(FrameworkHandle, testConverter, dumpXml))
                        {
                            try
                            {
                                _activeRunner.Run(listener, filter);
                            }
                            catch (NullReferenceException)
                            {
                                // this happens during the run when CancelRun is called.
                                TestLog.Debug("Nullref caught");
                            }
                        }
                    }
                }
                else
                {
                    var msgNode = loadResult.SelectSingleNode("properties/property[@name='_SKIPREASON']");
                    if (msgNode != null && (new[] { "contains no tests", "Has no TestFixtures" }).Any(msgNode.GetAttribute("value").Contains))
                    {
                        TestLog.Info("NUnit couldn't find any tests in " + assemblyPath);
                    }
                    else
                    {
                        TestLog.Info("NUnit failed to load " + assemblyPath);
                    }
                }
            }
            catch (BadImageFormatException)
            {
                // we skip the native c++ binaries that we don't support.
                TestLog.Warning("Assembly not supported: " + assemblyPath);
            }
            catch (FileNotFoundException ex)
            {
                // Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
                TestLog.Warning("Dependent Assembly " + ex.FileName + " of " + assemblyPath + " not found. Can be ignored if not a NUnit project.");
            }
            catch (Exception ex)
            {
                if (ex is TargetInvocationException)
                {
                    ex = ex.InnerException;
                }
                TestLog.Warning("Exception thrown executing tests in " + assemblyPath, ex);
            }
            finally
            {
#if !NETCOREAPP1_0
                dumpXml?.Dump4Execution();
#endif
                try
                {
                    _activeRunner?.Dispose();
                    _activeRunner = null;
                }
                catch (Exception ex)
                {
                    // can happen if CLR throws CannotUnloadAppDomainException, for example
                    // due to a long-lasting operation in a protected region (catch/finally clause).
                    if (ex is TargetInvocationException)
                    {
                        ex = ex.InnerException;
                    }
                    TestLog.Warning("Exception thrown unloading tests from " + assemblyPath, ex);
                }
            }
        }
Example #5
0
        private void RunAssembly(string assemblyName, TestFilter filter)
        {
#if LAUNCHDEBUGGER
            if (!Debugger.IsAttached)
            {
                Debugger.Launch();
            }
#endif
            _activeRunner = GetRunnerFor(assemblyName);

            try
            {
                var loadResult = _activeRunner.Explore(TestFilter.Empty);

                if (loadResult.Name == "test-run")
                {
                    loadResult = loadResult.FirstChild;
                }

                if (loadResult.GetAttribute("runstate") == "Runnable")
                {
                    var nunitTestCases = loadResult.SelectNodes("//test-case");

                    using (var testConverter = new TestConverter(TestLog, assemblyName))
                    {
                        var loadedTestCases = new List <TestCase>();

                        // As a side effect of calling TestConverter.ConvertTestCase,
                        // the converter's cache of all test cases is populated as well.
                        // All future calls to convert a test case may now use the cache.
                        foreach (XmlNode testNode in nunitTestCases)
                        {
                            loadedTestCases.Add(testConverter.ConvertTestCase(testNode));
                        }

                        TestLog.Info(string.Format("NUnit3TestExecutor converted {0} of {1} NUnit test cases", loadedTestCases.Count, nunitTestCases.Count));

                        // If we have a TFS Filter, convert it to an nunit filter
                        if (TfsFilter != null && !TfsFilter.IsEmpty)
                        {
                            // NOTE This overwrites filter used in call
                            var filterBuilder = new NUnitTestFilterBuilder(TestEngine.Services.GetService <ITestFilterService>());
                            filter = filterBuilder.ConvertTfsFilterToNUnitFilter(TfsFilter, loadedTestCases);
                        }

                        if (filter == NUnitTestFilterBuilder.NoTestsFound)
                        {
                            TestLog.Info("Skipping assembly - no matching test cases found");
                            return;
                        }

                        using (var listener = new NUnitEventListener(FrameworkHandle, testConverter))
                        {
                            try
                            {
                                _activeRunner.Run(listener, filter);
                            }
                            catch (NullReferenceException)
                            {
                                // this happens during the run when CancelRun is called.
                                TestLog.Debug("Nullref caught");
                            }
                        }
                    }
                }
                else
                {
                    TestLog.Info("NUnit failed to load " + assemblyName);
                }
            }
            catch (BadImageFormatException)
            {
                // we skip the native c++ binaries that we don't support.
                TestLog.Warning("Assembly not supported: " + assemblyName);
            }
            catch (System.IO.FileNotFoundException ex)
            {
                // Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
                TestLog.Warning("Dependent Assembly " + ex.FileName + " of " + assemblyName + " not found. Can be ignored if not a NUnit project.");
            }
            catch (Exception ex)
            {
                if (ex is TargetInvocationException)
                {
                    ex = ex.InnerException;
                }
                TestLog.Error("Exception thrown executing tests in " + assemblyName, ex);
            }

            _activeRunner.Dispose();
            _activeRunner = null;
        }
        public void RunAssembly(IFrameworkHandle testLog)
        {
            try
            {
#if LAUNCHDEBUGGER
            System.Diagnostics.Debugger.Launch();
#endif
                if (TryLoadAssembly())
                {
                    using (NUnitEventListener listener = new NUnitEventListener(testLog, TestConverter))
                    {
                        try
                        {
                            runner.Run(listener, NUnitFilter, true, LoggingThreshold.Off);
                        }
                        catch (NullReferenceException)
                        {
                            // this happens during the run when CancelRun is called.
                            logger.SendDebugMessage("Nullref caught");
                        }
                        finally
                        {
                            runner.Unload();
                        }
                    }
                }
                else
                {
                    logger.NUnitLoadError(assemblyName);
                }
            }
            catch (BadImageFormatException)
            {
                // we skip the native c++ binaries that we don't support.
                logger.AssemblyNotSupportedWarning(assemblyName);
            }
            catch (System.IO.FileNotFoundException ex)
            {
                // Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
                logger.DependentAssemblyNotFoundWarning(ex.FileName, assemblyName);
            }
            catch (Exception ex)
            {
                logger.SendErrorMessage("Exception thrown executing tests in " + assemblyName, ex);
            }
        }
Example #7
0
        private void RunAssembly(string assemblyName, IFrameworkHandle frameworkHandle)
        {
#if LAUNCHDEBUGGER
            if (!Debugger.IsAttached)
            {
                Debugger.Launch();
            }
#endif
            _testRunner = GetRunnerFor(assemblyName);

            try
            {
                var loadResult = _testRunner.Explore(TestFilter.Empty);

                if (loadResult.Name == "test-run")
                {
                    loadResult = loadResult.FirstChild;
                }

                if (loadResult.GetAttribute("runstate") == "Runnable")
                {
                    TestLog.SendInformationalMessage(string.Format("Loading tests from {0}", assemblyName));

                    var nunitTestCases = loadResult.SelectNodes("//test-case");

                    using (var testConverter = new TestConverter(TestLog, assemblyName))
                    {
                        var loadedTestCases = new List <TestCase>();

                        // As a side effect of calling TestConverter.ConvertTestCase,
                        // the converter's cache of all test cases is populated as well.
                        // All future calls to convert a test case may now use the cache.
                        foreach (XmlNode testNode in nunitTestCases)
                        {
                            loadedTestCases.Add(testConverter.ConvertTestCase(testNode));
                        }

                        // If we have a TFS Filter, convert it to an nunit filter
                        if (_tfsFilter != null && _tfsFilter.HasTfsFilterValue)
                        {
                            var filteredTestCases = _tfsFilter.CheckFilter(loadedTestCases);
                            var testCases         = filteredTestCases as TestCase[] ?? filteredTestCases.ToArray();
                            TestLog.SendInformationalMessage(string.Format("TFS Filter detected: LoadedTestCases {0}, Filterered Test Cases {1}", loadedTestCases.Count, testCases.Count()));
                            _nunitFilter = MakeTestFilter(testCases);
                        }

                        using (var listener = new NUnitEventListener(frameworkHandle, testConverter))
                        {
                            try
                            {
                                _testRunner.Run(listener, _nunitFilter);
                            }
                            catch (NullReferenceException)
                            {
                                // this happens during the run when CancelRun is called.
                                TestLog.SendDebugMessage("Nullref caught");
                            }
                        }
                    }
                }
                else
                {
                    TestLog.NUnitLoadError(assemblyName);
                }
            }
            catch (BadImageFormatException)
            {
                // we skip the native c++ binaries that we don't support.
                TestLog.AssemblyNotSupportedWarning(assemblyName);
            }
            catch (System.IO.FileNotFoundException ex)
            {
                // Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
                TestLog.DependentAssemblyNotFoundWarning(ex.FileName, assemblyName);
            }
            catch (Exception ex)
            {
                if (ex is TargetInvocationException)
                {
                    ex = ex.InnerException;
                }
                TestLog.SendErrorMessage("Exception thrown executing tests in " + assemblyName, ex);
            }
            _testRunner.Dispose();
        }
        private void RunAssembly(string assemblyPath, IGrouping <string, TestCase> testCases, TestFilter filter)
        {
#if LAUNCHDEBUGGER
            if (!Debugger.IsAttached)
            {
                Debugger.Launch();
            }
#endif

            string actionText    = Debugger.IsAttached ? "Debugging " : "Running ";
            string selectionText = filter == null || filter == TestFilter.Empty ? "all" : "selected";
            TestLog.Info(actionText + selectionText + " tests in " + assemblyPath);

            // No need to restore if the seed was in runsettings file
            if (!Settings.RandomSeedSpecified)
            {
                Settings.RestoreRandomSeed(Path.GetDirectoryName(assemblyPath));
            }
            executionDumpXml = null;
            if (Settings.DumpXmlTestResults)
            {
                executionDumpXml = new DumpXml(assemblyPath);
            }

            try
            {
                var package = CreateTestPackage(assemblyPath, testCases);
                NUnitEngineAdapter.CreateRunner(package);
                CreateTestOutputFolder();
                executionDumpXml?.AddString($"<NUnitDiscoveryInExecution>{assemblyPath}</NUnitExecution>\r\n\r\n");
                var discoveryResults = NUnitEngineAdapter.Explore(filter); // _activeRunner.Explore(filter);
                executionDumpXml?.AddString(discoveryResults.AsString());

                if (discoveryResults.IsRunnable)
                {
                    var nunitTestCases = discoveryResults.TestCases();

                    using var testConverter = new TestConverter(TestLog, assemblyPath, Settings);
                    var loadedTestCases = new List <TestCase>();

                    // As a side effect of calling TestConverter.ConvertTestCase,
                    // the converter's cache of all test cases is populated as well.
                    // All future calls to convert a test case may now use the cache.
                    foreach (XmlNode testNode in nunitTestCases)
                    {
                        loadedTestCases.Add(testConverter.ConvertTestCase(new NUnitTestCase(testNode)));
                    }


                    TestLog.Info($"   NUnit3TestExecutor converted {loadedTestCases.Count} of {nunitTestCases.Count} NUnit test cases");


                    // If we have a TFS Filter, convert it to an nunit filter
                    if (TfsFilter != null && !TfsFilter.IsEmpty)
                    {
                        // NOTE This overwrites filter used in call
                        var filterBuilder = CreateTestFilterBuilder();
                        filter = filterBuilder.ConvertTfsFilterToNUnitFilter(TfsFilter, loadedTestCases);
                    }

                    if (filter == NUnitTestFilterBuilder.NoTestsFound)
                    {
                        TestLog.Info("   Skipping assembly - no matching test cases found");
                        return;
                    }
                    executionDumpXml?.AddString($"<NUnitExecution>{assemblyPath}</NUnitExecution>\r\n");
                    using (var listener = new NUnitEventListener(FrameworkHandle, testConverter, this))
                    {
                        try
                        {
                            var results = NUnitEngineAdapter.Run(listener, filter);
                            GenerateTestOutput(results.TopNode, assemblyPath);
                        }
                        catch (NullReferenceException)
                        {
                            // this happens during the run when CancelRun is called.
                            TestLog.Debug("   Null ref caught");
                        }
                    }
                }
                else
                {
                    TestLog.Info(discoveryResults.HasNoNUnitTests
                            ? "   NUnit couldn't find any tests in " + assemblyPath
                            : "   NUnit failed to load " + assemblyPath);
                }
            }
            catch (BadImageFormatException)
            {
                // we skip the native c++ binaries that we don't support.
                TestLog.Warning("   Assembly not supported: " + assemblyPath);
            }
            catch (NUnitEngineException e)
            {
                if (e.InnerException is BadImageFormatException)
                {
                    // we skip the native c++ binaries that we don't support.
                    TestLog.Warning("   Assembly not supported: " + assemblyPath);
                }
                throw;
            }
            catch (FileNotFoundException ex)
            {
                // Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
                TestLog.Warning("   Dependent Assembly " + ex.FileName + " of " + assemblyPath + " not found. Can be ignored if not an NUnit project.");
            }
            catch (Exception ex)
            {
                if (ex is TargetInvocationException)
                {
                    ex = ex.InnerException;
                }
                TestLog.Warning("   Exception thrown executing tests in " + assemblyPath, ex);
            }
            finally
            {
                executionDumpXml?.Dump4Execution();
                try
                {
                    NUnitEngineAdapter?.CloseRunner();
                }
                catch (Exception ex)
                {
                    // can happen if CLR throws CannotUnloadAppDomainException, for example
                    // due to a long-lasting operation in a protected region (catch/finally clause).
                    if (ex is TargetInvocationException)
                    {
                        ex = ex.InnerException;
                    }
                    TestLog.Warning($"   Exception thrown unloading tests from {assemblyPath}", ex);
                }
            }
        }
        private void RunAssembly(string assemblyName, TestFilter filter)
        {
#if LAUNCHDEBUGGER
            if (!Debugger.IsAttached)
                Debugger.Launch();
#endif
            _activeRunner = GetRunnerFor(assemblyName);

            try
            {
                var loadResult = _activeRunner.Explore(TestFilter.Empty);

                if (loadResult.Name == "test-run")
                    loadResult = loadResult.FirstChild;

                if (loadResult.GetAttribute("runstate") == "Runnable")
                {
                    var nunitTestCases = loadResult.SelectNodes("//test-case");

                    using (var testConverter = new TestConverter(TestLog, assemblyName))
                    {
                        var loadedTestCases = new List<TestCase>();

                        // As a side effect of calling TestConverter.ConvertTestCase, 
                        // the converter's cache of all test cases is populated as well. 
                        // All future calls to convert a test case may now use the cache.
                        foreach (XmlNode testNode in nunitTestCases)
                            loadedTestCases.Add(testConverter.ConvertTestCase(testNode));

                        TestLog.Info(string.Format("NUnit3TestExecutor converted {0} of {1} NUnit test cases", loadedTestCases.Count, nunitTestCases.Count));

                        // If we have a TFS Filter, convert it to an nunit filter
                        if (TfsFilter != null && !TfsFilter.IsEmpty)
                        {
                            // NOTE This overwrites filter used in call
                            var filterBuilder = new NUnitTestFilterBuilder(TestEngine.Services.GetService<ITestFilterService>());
                            filter = filterBuilder.ConvertTfsFilterToNUnitFilter(TfsFilter, loadedTestCases);
                        }

                        if (filter == NUnitTestFilterBuilder.NoTestsFound)
                        {
                            TestLog.Info("Skipping assembly - no matching test cases found");
                            return;
                        }

                        using (var listener = new NUnitEventListener(FrameworkHandle, testConverter))
                        {
                            try
                            {
                                _activeRunner.Run(listener, filter);
                            }
                            catch (NullReferenceException)
                            {
                                // this happens during the run when CancelRun is called.
                                TestLog.Debug("Nullref caught");
                            }
                        }
                    }
                }
                else
                    TestLog.Info("NUnit failed to load " + assemblyName);
            }
            catch (BadImageFormatException)
            {
                // we skip the native c++ binaries that we don't support.
                TestLog.Warning("Assembly not supported: " + assemblyName);
            }
            catch (System.IO.FileNotFoundException ex)
            {
                // Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
                TestLog.Warning("Dependent Assembly " + ex.FileName + " of " + assemblyName + " not found. Can be ignored if not a NUnit project.");
            }
            catch (Exception ex)
            {
                if (ex is TargetInvocationException)
                    ex = ex.InnerException;
                TestLog.Error("Exception thrown executing tests in " + assemblyName, ex);
            }

            _activeRunner.Dispose();
            _activeRunner = null;
        }