Example #1
0
        public void ThatPathIsCorrectlyParsedInDiscoveryPhase(string path, string expected)
        {
            var file = Substitute.For <IFile>();
            var sut  = new DumpXml(path, file);

            sut.AddString("whatever");
            sut.DumpForDiscovery();
            file.Received().WriteAllText(Arg.Is <string>(o => o.StartsWith(expected, StringComparison.OrdinalIgnoreCase)), Arg.Any <string>());
        }
 private void SetupDump(string assemblyPath, IGrouping <string, TestCase> testCases)
 {
     executionDumpXml = null;
     if (Settings.DumpXmlTestResults)
     {
         executionDumpXml = new DumpXml(assemblyPath);
         string runningBy = testCases == null
             ? "<RunningBy>Sources</RunningBy>"
             : "<RunningBy>TestCases</RunningBy>";
         executionDumpXml?.AddString($"\n{runningBy}\n");
     }
 }
Example #3
0
        public void ThatEmptyContainsHeaders()
        {
            string res  = "";
            var    file = Substitute.For <IFile>();

            file.WriteAllText(Arg.Any <string>(), Arg.Do <string>(o => res = o));
            var sut = new DumpXml("Whatever", file);

            sut.DumpForDiscovery();
            Assert.That(res, Does.Contain("NUnitXml"));
            var sarray = res.Split('\n');

            Assert.That(sarray.Length, Is.GreaterThanOrEqualTo(3));
        }
Example #4
0
        public void ThatRandomNameContainsValidCharacters()
        {
            var sut = new DumpXml("whatever");
            var res = sut.RandomName();

            Assert.That(res, Does.EndWith(".dump"));
            var parts = res.Split('.');

            Assert.That(parts.Length, Is.EqualTo(2), $"Too many dots in {res}");
            var part1 = parts[0];
            var rg    = new Regex(@"^[a-zA-Z0-9\s]*$");

            Assert.That(rg.IsMatch(part1));
        }
Example #5
0
        public void ThatWritingClearsBuffer()
        {
            string res  = "";
            var    file = Substitute.For <IFile>();

            file.WriteAllText(Arg.Any <string>(), Arg.Do <string>(o => res = o));
            var sut     = new DumpXml("whatever", file);
            var string1 = "something";

            sut.AddString(string1);
            sut.DumpForDiscovery();
            Assert.That(res, Does.Contain(string1));
            var string2 = "new string";

            sut.AddString(string2);
            sut.DumpForDiscovery();
            Assert.That(res, Does.Contain(string2));
            Assert.That(res, Does.Not.Contain(string1));
        }
Example #6
0
        private void RunAssembly(string assemblyPath, TestFilter filter)
        {
#if LAUNCHDEBUGGER
            if (!Debugger.IsAttached)
            {
                Debugger.Launch();
            }
#endif

            var actionText    = Debugger.IsAttached ? "Debugging " : "Running ";
            var selectionText = filter == null || filter == TestFilter.Empty ? "all" : "selected";
            TestLog.Info(actionText + selectionText + " tests in " + assemblyPath);

            // No need to restore if the seed was in runsettings file
            if (!Settings.RandomSeedSpecified)
            {
                Settings.RestoreRandomSeed(Path.GetDirectoryName(assemblyPath));
            }
            DumpXml dumpXml = null;
            if (Settings.DumpXmlTestResults)
            {
                dumpXml = new Dump.DumpXml(assemblyPath);
            }

            try
            {
                _activeRunner = GetRunnerFor(assemblyPath);

                var loadResult = _activeRunner.Explore(TestFilter.Empty);
#if !NETCOREAPP1_0
                dumpXml?.AddString(loadResult.AsString());
#endif
                if (loadResult.Name == "test-run")
                {
                    loadResult = loadResult.FirstChild;
                }

                if (loadResult.GetAttribute("runstate") == "Runnable")
                {
                    var nunitTestCases = loadResult.SelectNodes("//test-case");

                    using (var testConverter = new TestConverter(TestLog, assemblyPath, Settings.CollectSourceInformation))
                    {
                        var loadedTestCases = new List <TestCase>();

                        // As a side effect of calling TestConverter.ConvertTestCase,
                        // the converter's cache of all test cases is populated as well.
                        // All future calls to convert a test case may now use the cache.
                        foreach (XmlNode testNode in nunitTestCases)
                        {
                            loadedTestCases.Add(testConverter.ConvertTestCase(testNode));
                        }


                        TestLog.Info($"NUnit3TestExecutor converted {loadedTestCases.Count} of {nunitTestCases.Count} NUnit test cases");

                        // If we have a TFS Filter, convert it to an nunit filter
                        if (TfsFilter != null && !TfsFilter.IsEmpty)
                        {
                            // NOTE This overwrites filter used in call
                            var filterBuilder = CreateTestFilterBuilder();
                            filter = filterBuilder.ConvertTfsFilterToNUnitFilter(TfsFilter, loadedTestCases);
                        }

                        if (filter == NUnitTestFilterBuilder.NoTestsFound)
                        {
                            TestLog.Info("Skipping assembly - no matching test cases found");
                            return;
                        }

                        using (var listener = new NUnitEventListener(FrameworkHandle, testConverter, dumpXml))
                        {
                            try
                            {
                                _activeRunner.Run(listener, filter);
                            }
                            catch (NullReferenceException)
                            {
                                // this happens during the run when CancelRun is called.
                                TestLog.Debug("Nullref caught");
                            }
                        }
                    }
                }
                else
                {
                    var msgNode = loadResult.SelectSingleNode("properties/property[@name='_SKIPREASON']");
                    if (msgNode != null && (new[] { "contains no tests", "Has no TestFixtures" }).Any(msgNode.GetAttribute("value").Contains))
                    {
                        TestLog.Info("NUnit couldn't find any tests in " + assemblyPath);
                    }
                    else
                    {
                        TestLog.Info("NUnit failed to load " + assemblyPath);
                    }
                }
            }
            catch (BadImageFormatException)
            {
                // we skip the native c++ binaries that we don't support.
                TestLog.Warning("Assembly not supported: " + assemblyPath);
            }
            catch (FileNotFoundException ex)
            {
                // Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
                TestLog.Warning("Dependent Assembly " + ex.FileName + " of " + assemblyPath + " not found. Can be ignored if not a NUnit project.");
            }
            catch (Exception ex)
            {
                if (ex is TargetInvocationException)
                {
                    ex = ex.InnerException;
                }
                TestLog.Warning("Exception thrown executing tests in " + assemblyPath, ex);
            }
            finally
            {
#if !NETCOREAPP1_0
                dumpXml?.Dump4Execution();
#endif
                try
                {
                    _activeRunner?.Dispose();
                    _activeRunner = null;
                }
                catch (Exception ex)
                {
                    // can happen if CLR throws CannotUnloadAppDomainException, for example
                    // due to a long-lasting operation in a protected region (catch/finally clause).
                    if (ex is TargetInvocationException)
                    {
                        ex = ex.InnerException;
                    }
                    TestLog.Warning("Exception thrown unloading tests from " + assemblyPath, ex);
                }
            }
        }
Example #7
0
        private void RunAssembly(string assemblyPath, IGrouping <string, TestCase> testCases, TestFilter filter)
        {
            LogActionAndSelection(assemblyPath, filter);
            RestoreRandomSeed(assemblyPath);
            Dump = DumpXml.CreateDump(assemblyPath, testCases, Settings);

            try
            {
                var package = CreateTestPackage(assemblyPath, testCases);
                NUnitEngineAdapter.CreateRunner(package);
                CreateTestOutputFolder();
                Dump?.StartDiscoveryInExecution(testCases, filter, package);

                // var discoveryResults = RunType == RunType.CommandLineCurrentNUnit ? null : NUnitEngineAdapter.Explore(filter);
                var discoveryResults = NUnitEngineAdapter.Explore(filter);
                Dump?.AddString(discoveryResults?.AsString() ?? " No discovery");

                if (discoveryResults?.IsRunnable ?? true)
                {
                    var discovery = new DiscoveryConverter(TestLog, Settings);
                    discovery.Convert(discoveryResults, assemblyPath);
                    if (!Settings.SkipExecutionWhenNoTests || discovery.AllTestCases.Any())
                    {
                        var ea = ExecutionFactory.Create(this);
                        ea.Run(filter, discovery, this);
                    }
                    else
                    {
                        TestLog.InfoNoTests(assemblyPath);
                    }
                }
                else
                {
                    TestLog.InfoNoTests(discoveryResults.HasNoNUnitTests, assemblyPath);
                }
            }
            catch (Exception ex) when(ex is BadImageFormatException || ex.InnerException is BadImageFormatException)
            {
                // we skip the native c++ binaries that we don't support.
                TestLog.Warning("   Assembly not supported: " + assemblyPath);
            }
            catch (FileNotFoundException ex)
            {
                // Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
                TestLog.Warning("   Dependent Assembly " + ex.FileName + " of " + assemblyPath +
                                " not found. Can be ignored if not an NUnit project.");
            }
            catch (Exception ex)
            {
                if (ex is TargetInvocationException)
                {
                    ex = ex.InnerException;
                }
                TestLog.Warning("   Exception thrown executing tests in " + assemblyPath, ex);
            }
            finally
            {
                Dump?.DumpForExecution();
                try
                {
                    NUnitEngineAdapter?.CloseRunner();
                }
                catch (Exception ex)
                {
                    // can happen if CLR throws CannotUnloadAppDomainException, for example
                    // due to a long-lasting operation in a protected region (catch/finally clause).
                    if (ex is TargetInvocationException)
                    {
                        ex = ex.InnerException;
                    }

                    TestLog.Warning($"   Exception thrown unloading tests from {assemblyPath}", ex);
                }
            }
        }
        public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger, ITestCaseDiscoverySink discoverySink)
        {
            Initialize(discoveryContext, messageLogger);
            CheckIfDebug();
            TestLog.Info($"NUnit Adapter {AdapterVersion}: Test discovery starting");

            // Ensure any channels registered by other adapters are unregistered
            CleanUpRegisteredChannels();

            if (Settings.InProcDataCollectorsAvailable && sources.Count() > 1)
            {
                TestLog.Error("Unexpected to discover tests in multiple assemblies when InProcDataCollectors specified in run configuration.");
                Unload();
                return;
            }

            foreach (string sourceAssembly in sources)
            {
                string sourceAssemblyPath = Path.IsPathRooted(sourceAssembly) ? sourceAssembly : Path.Combine(Directory.GetCurrentDirectory(), sourceAssembly);
                TestLog.Debug("Processing " + sourceAssembly);
                if (Settings.DumpXmlTestDiscovery)
                {
                    dumpXml = new DumpXml(sourceAssemblyPath);
                }

                try
                {
                    var package = CreateTestPackage(sourceAssemblyPath, null);
                    NUnitEngineAdapter.CreateRunner(package);
                    var results = NUnitEngineAdapter.Explore();
                    dumpXml?.AddString(results.AsString());

                    if (results.IsRunnable)
                    {
                        int cases;
                        using (var testConverter = new TestConverterForXml(TestLog, sourceAssemblyPath, Settings))
                        {
                            var timing = new TimingLogger(Settings, TestLog);
                            cases = ProcessTestCases(results, discoverySink, testConverter);
                            timing.LogTime("Discovery/Processing/Converting:");
                        }

                        TestLog.Debug($"Discovered {cases} test cases");
                        // Only save if seed is not specified in runsettings
                        // This allows workaround in case there is no valid
                        // location in which the seed may be saved.
                        if (cases > 0 && !Settings.RandomSeedSpecified)
                        {
                            Settings.SaveRandomSeed(Path.GetDirectoryName(sourceAssemblyPath));
                        }
                    }
                    else
                    {
                        if (results.HasNoNUnitTests)
                        {
                            if (Settings.Verbosity > 0)
                            {
                                TestLog.Info("Assembly contains no NUnit 3.0 tests: " + sourceAssembly);
                            }
                        }
                        else
                        {
                            TestLog.Info("NUnit failed to load " + sourceAssembly);
                        }
                    }
                }
                catch (NUnitEngineException e)
                {
                    if (e.InnerException is BadImageFormatException)
                    {
                        // we skip the native c++ binaries that we don't support.
                        TestLog.Warning("Assembly not supported: " + sourceAssembly);
                    }
                    else
                    {
                        TestLog.Warning("Exception thrown discovering tests in " + sourceAssembly, e);
                    }
                }
                catch (BadImageFormatException)
                {
                    // we skip the native c++ binaries that we don't support.
                    TestLog.Warning("Assembly not supported: " + sourceAssembly);
                }
                catch (FileNotFoundException ex)
                {
                    // Either the NUnit framework was not referenced by the test assembly
                    // or some other error occurred. Not a problem if not an NUnit assembly.
                    TestLog.Warning("Dependent Assembly " + ex.FileName + " of " + sourceAssembly + " not found. Can be ignored if not an NUnit project.");
                }
                catch (FileLoadException ex)
                {
                    // Attempts to load an invalid assembly, or an assembly with missing dependencies
                    TestLog.Warning("Assembly " + ex.FileName + " loaded through " + sourceAssembly + " failed. Assembly is ignored. Correct deployment of dependencies if this is an error.");
                }
                catch (TypeLoadException ex)
                {
                    if (ex.TypeName == "NUnit.Framework.Api.FrameworkController")
                    {
                        TestLog.Warning("   Skipping NUnit 2.x test assembly");
                    }
                    else
                    {
                        TestLog.Warning("Exception thrown discovering tests in " + sourceAssembly, ex);
                    }
                }
                catch (Exception ex)
                {
                    TestLog.Warning("Exception thrown discovering tests in " + sourceAssembly, ex);
                }
                finally
                {
                    dumpXml?.DumpForDiscovery();
                    NUnitEngineAdapter?.CloseRunner();
                }
            }

            TestLog.Info($"NUnit Adapter {AdapterVersion}: Test discovery complete");

            Unload();
        }
        private void RunAssembly(string assemblyPath, IGrouping <string, TestCase> testCases, TestFilter filter)
        {
#if LAUNCHDEBUGGER
            if (!Debugger.IsAttached)
            {
                Debugger.Launch();
            }
#endif

            string actionText    = Debugger.IsAttached ? "Debugging " : "Running ";
            string selectionText = filter == null || filter == TestFilter.Empty ? "all" : "selected";
            TestLog.Info(actionText + selectionText + " tests in " + assemblyPath);

            // No need to restore if the seed was in runsettings file
            if (!Settings.RandomSeedSpecified)
            {
                Settings.RestoreRandomSeed(Path.GetDirectoryName(assemblyPath));
            }
            executionDumpXml = null;
            if (Settings.DumpXmlTestResults)
            {
                executionDumpXml = new DumpXml(assemblyPath);
            }

            try
            {
                var package = CreateTestPackage(assemblyPath, testCases);
                NUnitEngineAdapter.CreateRunner(package);
                CreateTestOutputFolder();
                executionDumpXml?.AddString($"<NUnitDiscoveryInExecution>{assemblyPath}</NUnitExecution>\r\n\r\n");
                var discoveryResults = NUnitEngineAdapter.Explore(filter); // _activeRunner.Explore(filter);
                executionDumpXml?.AddString(discoveryResults.AsString());

                if (discoveryResults.IsRunnable)
                {
                    var nunitTestCases = discoveryResults.TestCases();

                    using var testConverter = new TestConverter(TestLog, assemblyPath, Settings);
                    var loadedTestCases = new List <TestCase>();

                    // As a side effect of calling TestConverter.ConvertTestCase,
                    // the converter's cache of all test cases is populated as well.
                    // All future calls to convert a test case may now use the cache.
                    foreach (XmlNode testNode in nunitTestCases)
                    {
                        loadedTestCases.Add(testConverter.ConvertTestCase(new NUnitTestCase(testNode)));
                    }


                    TestLog.Info($"   NUnit3TestExecutor converted {loadedTestCases.Count} of {nunitTestCases.Count} NUnit test cases");


                    // If we have a TFS Filter, convert it to an nunit filter
                    if (TfsFilter != null && !TfsFilter.IsEmpty)
                    {
                        // NOTE This overwrites filter used in call
                        var filterBuilder = CreateTestFilterBuilder();
                        filter = filterBuilder.ConvertTfsFilterToNUnitFilter(TfsFilter, loadedTestCases);
                    }

                    if (filter == NUnitTestFilterBuilder.NoTestsFound)
                    {
                        TestLog.Info("   Skipping assembly - no matching test cases found");
                        return;
                    }
                    executionDumpXml?.AddString($"<NUnitExecution>{assemblyPath}</NUnitExecution>\r\n");
                    using (var listener = new NUnitEventListener(FrameworkHandle, testConverter, this))
                    {
                        try
                        {
                            var results = NUnitEngineAdapter.Run(listener, filter);
                            GenerateTestOutput(results.TopNode, assemblyPath);
                        }
                        catch (NullReferenceException)
                        {
                            // this happens during the run when CancelRun is called.
                            TestLog.Debug("   Null ref caught");
                        }
                    }
                }
                else
                {
                    TestLog.Info(discoveryResults.HasNoNUnitTests
                            ? "   NUnit couldn't find any tests in " + assemblyPath
                            : "   NUnit failed to load " + assemblyPath);
                }
            }
            catch (BadImageFormatException)
            {
                // we skip the native c++ binaries that we don't support.
                TestLog.Warning("   Assembly not supported: " + assemblyPath);
            }
            catch (NUnitEngineException e)
            {
                if (e.InnerException is BadImageFormatException)
                {
                    // we skip the native c++ binaries that we don't support.
                    TestLog.Warning("   Assembly not supported: " + assemblyPath);
                }
                throw;
            }
            catch (FileNotFoundException ex)
            {
                // Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
                TestLog.Warning("   Dependent Assembly " + ex.FileName + " of " + assemblyPath + " not found. Can be ignored if not an NUnit project.");
            }
            catch (Exception ex)
            {
                if (ex is TargetInvocationException)
                {
                    ex = ex.InnerException;
                }
                TestLog.Warning("   Exception thrown executing tests in " + assemblyPath, ex);
            }
            finally
            {
                executionDumpXml?.Dump4Execution();
                try
                {
                    NUnitEngineAdapter?.CloseRunner();
                }
                catch (Exception ex)
                {
                    // can happen if CLR throws CannotUnloadAppDomainException, for example
                    // due to a long-lasting operation in a protected region (catch/finally clause).
                    if (ex is TargetInvocationException)
                    {
                        ex = ex.InnerException;
                    }
                    TestLog.Warning($"   Exception thrown unloading tests from {assemblyPath}", ex);
                }
            }
        }