Assembly LoadAssembly(AssemblyLoadContext context, AssemblyName assemblyName) { if (lookupCache.TryGetValue(assemblyName.Name, out var result)) { return(result); } if (internalDiagnosticsMessageSink != null) { internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[AssemblyHelper_NetCoreApp.LoadAssembly] Resolving '{assemblyName.Name}'")); } var path = Path.Combine(directory, assemblyName.Name); result = ResolveAndLoadAssembly(context, path, out var resolvedAssemblyPath); if (internalDiagnosticsMessageSink != null) { if (result == null) { internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[AssemblyHelper_NetCoreApp.LoadAssembly] Resolution failed, passed down to next resolver")); } else { internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[AssemblyHelper_NetCoreApp.LoadAssembly] Successful: '{resolvedAssemblyPath}'")); } } lookupCache[assemblyName.Name] = result; return(result); }
public async Task InstallingSamePackageFromRemoteUpdatesLocal(string installCommandName, string uninstallCommandName) { _messageSink.OnMessage(new DiagnosticMessage($"{nameof(InstallingSamePackageFromRemoteUpdatesLocal)} started.")); var home = TestUtils.CreateTemporaryFolder("Home"); using var packageManager = new PackageManager(); string packageLocation = await packageManager.GetNuGetPackage( "Microsoft.DotNet.Common.ProjectTemplates.5.0", minimumVersion : new NuGet.Versioning.NuGetVersion(6, 0, 0), logger : new XunitNuGetLogger(_messageSink)).ConfigureAwait(false); _messageSink.OnMessage(new DiagnosticMessage($"{nameof(InstallingSamePackageFromRemoteUpdatesLocal)}: Microsoft.DotNet.Common.ProjectTemplates.5.0 is downloaded to {packageLocation}.)")); new DotnetNewCommand(_log, installCommandName, packageLocation) .WithCustomHive(home) .WithWorkingDirectory(TestUtils.CreateTemporaryFolder()) .Execute() .Should().ExitWith(0) .And.NotHaveStdErr() .And.HaveStdOutMatching($"Success: Microsoft\\.DotNet\\.Common\\.ProjectTemplates\\.5\\.0::([\\d\\.a-z-])+ installed the following templates:") .And.HaveStdOutContaining("console") .And.HaveStdOutContaining("classlib"); new DotnetNewCommand(_log, uninstallCommandName) .WithCustomHive(home) .WithWorkingDirectory(TestUtils.CreateTemporaryFolder()) .Execute() .Should().ExitWith(0) .And.NotHaveStdErr() .And.HaveStdOutContaining("Microsoft.DotNet.Common.ProjectTemplates.5.0") .And.HaveStdOutContaining("Author: Microsoft") .And.HaveStdOutContaining("Version:") .And.NotHaveStdOutContaining("Version: 5.0.0"); new DotnetNewCommand(_log, installCommandName, "Microsoft.DotNet.Common.ProjectTemplates.5.0::5.0.0") .WithCustomHive(home) .WithWorkingDirectory(TestUtils.CreateTemporaryFolder()) .Execute() .Should().ExitWith(0) .And.NotHaveStdErr() .And.HaveStdOutContaining("The following template packages will be installed:") .And.HaveStdOutContaining("Microsoft.DotNet.Common.ProjectTemplates.5.0::5.0.0") .And.HaveStdOutMatching("Microsoft\\.DotNet\\.Common\\.ProjectTemplates\\.5\\.0 \\(version ([\\d\\.a-z-])+\\) is already installed, it will be replaced with version 5\\.0\\.0") .And.HaveStdOutMatching("Microsoft\\.DotNet\\.Common\\.ProjectTemplates\\.5\\.0::([\\d\\.a-z-])+ was successfully uninstalled") .And.HaveStdOutContaining($"Success: Microsoft.DotNet.Common.ProjectTemplates.5.0::5.0.0 installed the following templates:") .And.HaveStdOutContaining("console") .And.HaveStdOutContaining("classlib"); new DotnetNewCommand(_log, uninstallCommandName) .WithCustomHive(home) .WithWorkingDirectory(TestUtils.CreateTemporaryFolder()) .Execute() .Should().ExitWith(0) .And.NotHaveStdErr() .And.HaveStdOutContaining("Microsoft.DotNet.Common.ProjectTemplates.5.0") .And.HaveStdOutContaining("Author: Microsoft") .And.HaveStdOutContaining("Version: 5.0.0"); _messageSink.OnMessage(new DiagnosticMessage($"{nameof(InstallingSamePackageFromRemoteUpdatesLocal)} finished.")); }
RunSummary RunTestClass(ITestCollection testCollection, string className, IList <Xunit1TestCase> testCases, IMessageSink messageSink) { var handler = new TestClassCallbackHandler(testCases, messageSink); var results = handler.TestClassResults; results.Continue = messageSink.OnMessage(new TestClassStarting(testCollection, className)); if (results.Continue) { try { var methodNames = testCases.Select(tc => tc.Method.Name).ToList(); executor.RunTests(className, methodNames, handler); handler.LastNodeArrived.WaitOne(); } catch (Exception ex) { var failureInformation = Xunit1ExceptionUtility.ConvertToFailureInformation(ex); results.Continue = messageSink.OnMessage(new ErrorMessage(failureInformation.ExceptionTypes, failureInformation.Messages, failureInformation.StackTraces, failureInformation.ExceptionParentIndices)) && results.Continue; } } results.Continue = messageSink.OnMessage(new TestClassFinished(testCollection, className, results.Time, results.Total, results.Failed, results.Skipped)) && results.Continue; return(results); }
bool OnStart(XmlNode xml) { var testCase = FindTestCase(xml.Attributes["type"].Value, xml.Attributes["method"].Value); SendTestCaseMessagesWhenAppropriate(testCase); return(messageSink.OnMessage(new TestStarting(testCase, xml.Attributes["name"].Value)) && TestClassResults.Continue); }
bool OnClass(XmlNode xml) { SendTestCaseMessagesWhenAppropriate(null); var @continue = true; XmlNode failureNode; if ((failureNode = xml.SelectSingleNode("failure")) != null) { var failureInformation = Xunit1ExceptionUtility.ConvertToFailureInformation(failureNode); var errorMessage = new ErrorMessage(testCases, failureInformation.ExceptionTypes, failureInformation.Messages, failureInformation.StackTraces, failureInformation.ExceptionParentIndices); @continue = messageSink.OnMessage(errorMessage); } TestClassResults.Time = Decimal.Parse(xml.Attributes["time"].Value, CultureInfo.InvariantCulture); TestClassResults.Total = Int32.Parse(xml.Attributes["total"].Value, CultureInfo.InvariantCulture); TestClassResults.Failed = Int32.Parse(xml.Attributes["failed"].Value, CultureInfo.InvariantCulture); TestClassResults.Skipped = Int32.Parse(xml.Attributes["skipped"].Value, CultureInfo.InvariantCulture); return(@continue && TestClassResults.Continue); }
public OutputController(IReportConfiguration configuration, IMessageSink diagnosticMessageSink) { _configuration = configuration; _diagnosticMessageSink = diagnosticMessageSink; IReportWriter writer; if (!configuration.WriteOutput) { _diagnosticMessageSink.OnMessage(new DiagnosticMessage("Output is disabled")); writer = new NullWriter(); } else { _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Creating report at {configuration.XmlReportFile}")); _fileStream = new FileStream(configuration.XmlReportFile, FileMode.Create, FileAccess.Write, FileShare.None, 4096, true); _sw = new StreamWriter(_fileStream); XmlWriterSettings xws = new XmlWriterSettings(); xws.Async = true; _xw = XmlWriter.Create(_sw, xws); writer = new ReportWriter(_xw); } Report = new ScenarioReport(configuration.AssemblyName, writer, _diagnosticMessageSink); }
/// <inheritdoc/> public void Run(IEnumerable<ITestCase> testMethods, IMessageSink messageSink) { bool cancelled = false; var totalSummary = new RunSummary(); string currentDirectory = Directory.GetCurrentDirectory(); try { Directory.SetCurrentDirectory(Path.GetDirectoryName(assemblyInfo.AssemblyPath)); if (messageSink.OnMessage(new TestAssemblyStarting())) { var classGroups = testMethods.Cast<XunitTestCase>().GroupBy(tc => tc.Class).ToList(); if (classGroups.Count > 0) { var collectionSummary = new RunSummary(); if (messageSink.OnMessage(new TestCollectionStarting())) { foreach (var group in classGroups) { var classSummary = new RunSummary(); if (!messageSink.OnMessage(new TestClassStarting { ClassName = group.Key.Name })) cancelled = true; else { cancelled = RunTestClass(messageSink, group, classSummary); collectionSummary.Aggregate(classSummary); } if (!messageSink.OnMessage(new TestClassFinished { Assembly = assemblyInfo, ClassName = group.Key.Name, TestsRun = classSummary.Total })) cancelled = true; if (cancelled) break; } } messageSink.OnMessage(new TestCollectionFinished { Assembly = assemblyInfo, TestsRun = collectionSummary.Total }); totalSummary.Aggregate(collectionSummary); } } messageSink.OnMessage(new TestAssemblyFinished { Assembly = assemblyInfo, TestsRun = totalSummary.Total, TestsFailed = totalSummary.Failed, TestsSkipped = totalSummary.Skipped, ExecutionTime = totalSummary.Time }); } finally { Directory.SetCurrentDirectory(currentDirectory); } }
public Assembly LoadManagedDll([NotNull] string assemblyName, [NotNull] Func <string, Assembly> managedAssemblyLoader) { if (_managedAssemblyCache.TryGetValue(assemblyName, out var result)) { return(result); } _internalDiagnosticsMessageSink?.OnMessage(new DiagnosticMessage($"[DependencyContextAssemblyCache.LoadManagedDll] Resolving '{assemblyName}'")); var tupleResult = ResolveManagedAssembly(assemblyName, managedAssemblyLoader); var resolvedAssemblyPath = tupleResult.Item1; result = tupleResult.Item2; _managedAssemblyCache[assemblyName] = result; if (_internalDiagnosticsMessageSink == null) { return(result); } _internalDiagnosticsMessageSink.OnMessage( result == null ? new DiagnosticMessage("[DependencyContextAssemblyCache.LoadManagedDll] Resolution failed, passed down to next resolver") : new DiagnosticMessage($"[DependencyContextAssemblyCache.LoadManagedDll] Successful: '{resolvedAssemblyPath}'")); return(result); }
/// <inheritdoc/> protected override IntPtr LoadUnmanagedDll(string unmanagedDllName) { var result = base.LoadUnmanagedDll(unmanagedDllName); if (result != default) { return(result); } if (!unmanagedAssemblyCache.TryGetValue(unmanagedDllName, out var resolvedAssemblyPath)) { if (internalDiagnosticsMessageSink != null) { internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[NetCoreAssemblyDependencyResolver.LoadUnmanagedDll] Resolving '{unmanagedDllName}'")); } resolvedAssemblyPath = ResolveUnmanagedAssembly(unmanagedDllName); unmanagedAssemblyCache[unmanagedDllName] = resolvedAssemblyPath; if (internalDiagnosticsMessageSink != null) { if (resolvedAssemblyPath == null) { internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[NetCoreAssemblyDependencyResolver.LoadUnmanagedDll] Resolution failed, passed down to next resolver")); } else { internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[NetCoreAssemblyDependencyResolver.LoadUnmanagedDll] Successful: '{resolvedAssemblyPath}'")); } } } return(resolvedAssemblyPath != null?LoadUnmanagedDllFromPath(resolvedAssemblyPath) : default);
void Find(Predicate <ITestCaseDiscoveryMessage> filter, bool includeSourceInformation, IMessageSink messageSink) { try { XmlNode assemblyXml = null; var handler = new XmlNodeCallbackHandler(xml => { assemblyXml = xml; return(true); }); executor.EnumerateTests(handler); foreach (XmlNode method in assemblyXml.SelectNodes("//method")) { var testCase = method.ToTestCase(assemblyFileName, configFileName); if (testCase != null) { if (includeSourceInformation) { testCase.SourceInformation = sourceInformationProvider.GetSourceInformation(testCase); } var message = new TestCaseDiscoveryMessage(testCase); if (filter(message)) { messageSink.OnMessage(message); } } } } finally { messageSink.OnMessage(new DiscoveryCompleteMessage()); } }
protected override async Task <RunSummary> RunTestCaseAsync(IXunitTestCase testCase) { var parameters = string.Empty; if (testCase.TestMethodArguments != null) { parameters = string.Join(", ", testCase.TestMethodArguments.Select(a => a?.ToString() ?? "null")); } var test = $"{TestMethod.TestClass.Class.Name}.{TestMethod.Method.Name}({parameters})"; _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"STARTED: {test}")); using var timer = new Timer( _ => _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"WARNING: {test} has been running for more than 15 minutes")), null, TimeSpan.FromMinutes(15), Timeout.InfiniteTimeSpan); try { var result = await base.RunTestCaseAsync(testCase); var status = result.Failed > 0 ? "FAILURE" : "SUCCESS"; _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"{status}: {test} ({result.Time}s)")); return(result); } catch (Exception ex) { _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"ERROR: {test} ({ex.Message})")); throw; } }
Assembly LoadAssembly(AssemblyName assemblyName) { if (lookupCache.TryGetValue(assemblyName.Name, out var result)) { return(result); } var path = Path.Combine(directory, assemblyName.Name); result = ResolveAndLoadAssembly(path, out var resolvedAssemblyPath); if (internalDiagnosticsMessageSink != null) { if (result == null) { internalDiagnosticsMessageSink.OnMessage(new _DiagnosticMessage($"[AssemblyHelper_Desktop.LoadAssembly] Resolution for '{assemblyName.Name}' failed, passed down to next resolver")); } else { internalDiagnosticsMessageSink.OnMessage(new _DiagnosticMessage($"[AssemblyHelper_Desktop.LoadAssembly] Resolved '{assemblyName.Name}' to '{resolvedAssemblyPath}'")); } } lookupCache[assemblyName.Name] = result; return(result); }
/// <summary> /// Starts the process of running all the xUnit.net v1 tests. /// </summary> /// <param name="testCases">The test cases to run; if null, all tests in the assembly are run.</param> /// <param name="messageSink">The message sink to report results back to.</param> public void Run(IEnumerable <ITestCase> testCases, IMessageSink messageSink) { var results = new Xunit1RunSummary(); var environment = String.Format("{0}-bit .NET {1}", IntPtr.Size * 8, Environment.Version); var firstTestCase = testCases.FirstOrDefault(); var testCollection = firstTestCase == null ? null : firstTestCase.TestMethod.TestClass.TestCollection; if (testCollection != null) { try { if (messageSink.OnMessage(new TestAssemblyStarting(testCases, testCollection.TestAssembly, DateTime.Now, environment, TestFrameworkDisplayName))) { results = RunTestCollection(testCollection, testCases, messageSink); } } catch (Exception ex) { var failureInformation = Xunit1ExceptionUtility.ConvertToFailureInformation(ex); messageSink.OnMessage(new ErrorMessage(testCases, failureInformation.ExceptionTypes, failureInformation.Messages, failureInformation.StackTraces, failureInformation.ExceptionParentIndices)); } finally { messageSink.OnMessage(new TestAssemblyFinished(testCases, testCollection.TestAssembly, results.Time, results.Total, results.Failed, results.Skipped)); } } }
/// <summary> /// Gives an opportunity to override test case orderer. By default, this method gets the /// orderer from the collection definition. If this function returns <c>null</c>, the /// test case orderer passed into the constructor will be used. /// </summary> protected virtual ITestCaseOrderer GetTestCaseOrderer() { if (TestCollection.CollectionDefinition != null) { var ordererAttribute = TestCollection.CollectionDefinition.GetCustomAttributes(typeof(TestCaseOrdererAttribute)).SingleOrDefault(); if (ordererAttribute != null) { try { var testCaseOrderer = ExtensibilityPointFactory.GetTestCaseOrderer(diagnosticMessageSink, ordererAttribute); if (testCaseOrderer != null) { return(testCaseOrderer); } var args = ordererAttribute.GetConstructorArguments().Cast <string>().ToList(); diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Could not find type '{args[0]}' in {args[1]} for collection-level test case orderer on test collection '{TestCollection.DisplayName}'")); } catch (Exception ex) { var innerEx = ex.Unwrap(); var args = ordererAttribute.GetConstructorArguments().Cast <string>().ToList(); diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Collection-level test case orderer '{args[0]}' for test collection '{TestCollection.DisplayName}' threw '{innerEx.GetType().FullName}' during construction: {innerEx.Message}{Environment.NewLine}{innerEx.StackTrace}")); } } } return(null); }
Xunit1RunSummary RunTestCollection(ITestCollection testCollection, IEnumerable <ITestCase> testCases, IMessageSink messageSink) { var results = new Xunit1RunSummary(); results.Continue = messageSink.OnMessage(new TestCollectionStarting(testCases, testCollection)); try { if (results.Continue) { foreach (var testClassGroup in testCases.GroupBy(tc => tc.TestMethod.TestClass, Comparer.Instance)) { var classResults = RunTestClass(testClassGroup.Key, testClassGroup.ToList(), messageSink); results.Aggregate(classResults); if (!classResults.Continue) { break; } } } } finally { results.Continue = messageSink.OnMessage(new TestCollectionFinished(testCases, testCollection, results.Time, results.Total, results.Failed, results.Skipped)) && results.Continue; } return(results); }
/// <inheritdoc/> protected void SetTestCaseOrderer(IReflectionTypeInfo Class) { var ordererAttribute = Class.GetCustomAttributes(typeof(TestCaseOrdererAttribute)).SingleOrDefault(); if (ordererAttribute != null) { try { var testCaseOrderer = ExtensibilityPointFactory.GetTestCaseOrderer(diagnosticMessageSink, ordererAttribute); if (testCaseOrderer != null) { TestCaseOrderer = testCaseOrderer; } else { var args = ordererAttribute.GetConstructorArguments().Cast <string>().ToList(); diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Could not find type '{args[0]}' in {args[1]} for class-level test case orderer on test class '{Class.Name}'")); } } catch (Exception ex) { var innerEx = ex.InnerException ?? ex; //.Unwrap(); var args = ordererAttribute.GetConstructorArguments().Cast <string>().ToList(); diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Class-level test case orderer '{args[0]}' for test class '{Class.Name}' threw '{innerEx.GetType().FullName}' " + $"during construction: {innerEx.Message}{Environment.NewLine}{innerEx.StackTrace}")); } } }
/// <inheritdoc/> protected override bool RunTests(IMessageSink messageSink, object[] constructorArguments, ExceptionAggregator aggregator) { bool cancelled = false; if (!messageSink.OnMessage(new TestStarting { TestCase = this, TestDisplayName = DisplayName })) cancelled = true; else { try { lambda(); if (!messageSink.OnMessage(new TestPassed { TestCase = this, TestDisplayName = DisplayName })) cancelled = true; } catch (Exception ex) { if (!messageSink.OnMessage(new TestFailed(ex) { TestCase = this, TestDisplayName = DisplayName })) cancelled = true; } } if (!messageSink.OnMessage(new TestFinished { TestCase = this, TestDisplayName = DisplayName })) cancelled = true; return cancelled; }
public static IDisposable SubscribeResolveForAssembly([NotNull] string assemblyFileName, [CanBeNull] IMessageSink internalDiagnosticsMessageSink) { var assemblyFolder = Path.GetDirectoryName(assemblyFileName).ThrowIfNull(); var depsJsonFile = Path.Combine(assemblyFolder, Path.GetFileNameWithoutExtension(assemblyFileName) + ".deps.json"); if (!File.Exists(depsJsonFile)) { internalDiagnosticsMessageSink?.OnMessage( new DiagnosticMessage( $"[DependencyContextAssemblyHelper.SubscribeResolveForAssembly] Skipping resolution for '{depsJsonFile}': File not found")); return(null); } using (var stream = File.OpenRead(depsJsonFile)) { var context = JsonReader.Read(stream); if (context != null) { return(new DependencyContextAssemblyHelper(assemblyFolder, context, internalDiagnosticsMessageSink)); } internalDiagnosticsMessageSink?.OnMessage( new DiagnosticMessage( $"[DependencyContextAssemblyHelper.SubscribeResolveForAssembly] Skipping resolution for '{depsJsonFile}': File appears to be malformed")); return(null); } }
protected override async Task <RunSummary> RunTestCaseAsync(IXunitTestCase testCase) { var parameters = string.Empty; if (testCase.TestMethodArguments != null) { parameters = string.Join(", ", testCase.TestMethodArguments.Select(a => a?.ToString() ?? "null")); } var test = $"{TestMethod.TestClass.Class.Name}.{TestMethod.Method.Name}({parameters})"; _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"STARTED: {test}")); try { var result = await base.RunTestCaseAsync(testCase); var status = result.Failed > 0 ? "FAILURE" : "SUCCESS"; _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"{status}: {test} ({result.Time}s)")); return(result); } catch (Exception ex) { _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"ERROR: {test} ({ex.Message})")); throw; } }
static Type GetTestFrameworkType(IAssemblyInfo testAssembly, IMessageSink diagnosticMessageSink) { try { var testFrameworkAttr = testAssembly.GetCustomAttributes(typeof(ITestFrameworkAttribute)).FirstOrDefault(); if (testFrameworkAttr != null) { var discovererAttr = testFrameworkAttr.GetCustomAttributes(typeof(TestFrameworkDiscovererAttribute)).FirstOrDefault(); if (discovererAttr != null) { var discoverer = ExtensibilityPointFactory.GetTestFrameworkTypeDiscoverer(diagnosticMessageSink, discovererAttr); if (discoverer != null) { return(discoverer.GetTestFrameworkType(testFrameworkAttr)); } var ctorArgs = discovererAttr.GetConstructorArguments().ToArray(); diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Unable to create custom test framework discoverer type '{ctorArgs[1]}, {ctorArgs[0]}'")); } else { diagnosticMessageSink.OnMessage(new DiagnosticMessage("Assembly-level test framework attribute was not decorated with [TestFrameworkDiscoverer]")); } } } catch (Exception ex) { diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Exception thrown during test framework discoverer construction: {ex.Unwrap()}")); } return(typeof(XunitTestFramework)); }
protected override async Task <RunSummary> RunTestAsync() { var runSummary = new BenchmarkRunSummary { TestClassFullName = TestCase.TestMethod.TestClass.Class.Name, TestClass = TestCase.TestMethod.TestClass.Class.Name.Split('.').Last(), TestMethod = TestCase.TestMethodName, Variation = TestCase.Variation, ProductReportingVersion = BenchmarkConfig.Instance.ProductReportingVersion, RunStarted = DateTime.UtcNow, MachineName = _machineName, Framework = _framework, Architecture = IntPtr.Size > 4 ? "x64" : "x86", WarmupIterations = TestCase.WarmupIterations, Iterations = TestCase.Iterations, CustomData = BenchmarkConfig.Instance.CustomData }; for (var i = 0; i < TestCase.WarmupIterations; i++) { var runner = CreateRunner(i + 1, TestCase.WarmupIterations, TestCase.Variation, warmup: true); runSummary.Aggregate(await runner.RunAsync()); } for (var i = 0; i < TestCase.Iterations; i++) { TestCase.MetricCollector.Reset(); var runner = CreateRunner(i + 1, TestCase.Iterations, TestCase.Variation, warmup: false); var iterationSummary = new BenchmarkIterationSummary(); iterationSummary.Aggregate(await runner.RunAsync(), TestCase.MetricCollector); runSummary.Aggregate(iterationSummary); } if (runSummary.Failed != 0) { _diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage($"No valid results for {TestCase.DisplayName}. {runSummary.Failed} of {TestCase.Iterations + TestCase.WarmupIterations} iterations failed.")); } else { runSummary.PopulateMetrics(); _diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage(runSummary.ToString())); foreach (var database in BenchmarkConfig.Instance.ResultDatabases) { try { new SqlServerBenchmarkResultProcessor(database).SaveSummary(runSummary); } catch (Exception ex) { _diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage($"Failed to save results to {database}{Environment.NewLine} {ex}")); throw; } } } return(runSummary); }
/// <summary> /// Initializes a new instance of the <see cref="DependencyContextAssemblyCache" /> class. /// </summary> /// <param name="assemblyFolder">The assembly folder</param> /// <param name="dependencyContext">The dependency context</param> /// <param name="internalDiagnosticsMessageSink">The message sink</param> public DependencyContextAssemblyCache( [NotNull] string assemblyFolder, [NotNull] DependencyContext dependencyContext, [CanBeNull] IMessageSink internalDiagnosticsMessageSink) { _assemblyFolder = assemblyFolder; _internalDiagnosticsMessageSink = internalDiagnosticsMessageSink; _assemblyResolver = new XunitPackageCompilationAssemblyResolver(internalDiagnosticsMessageSink); internalDiagnosticsMessageSink?.OnMessage( new DiagnosticMessage( $"[DependencyContextAssemblyCache..ctor] Runtime graph: [{string.Join(",", dependencyContext.RuntimeGraph.Select(x => $"'{x.Runtime}'"))}]")); var currentRuntime = RuntimeEnvironment.GetRuntimeIdentifier(); var fallbacks = dependencyContext.RuntimeGraph.FirstOrDefault(x => string.Equals(x.Runtime, currentRuntime, StringComparison.OrdinalIgnoreCase)); HashSet <string> compatibleRuntimes = fallbacks != null ? new HashSet <string>(fallbacks.Fallbacks, StringComparer.OrdinalIgnoreCase) : new HashSet <string>(StringComparer.OrdinalIgnoreCase); compatibleRuntimes.Add(currentRuntime); compatibleRuntimes.Add(string.Empty); internalDiagnosticsMessageSink?.OnMessage( new DiagnosticMessage( $"[DependencyContextAssemblyCache..ctor] Compatible runtimes: [{string.Join(",", compatibleRuntimes.Select(x => $"'{x}'"))}]")); _managedAssemblyCache = new Dictionary <string, Assembly>(StringComparer.OrdinalIgnoreCase); _managedAssemblyMap = dependencyContext.RuntimeLibraries .Where(lib => lib.RuntimeAssemblyGroups?.Count > 0) .Select(lib => Tuple.Create(lib, lib.RuntimeAssemblyGroups.FirstOrDefault(libGroup => compatibleRuntimes.Contains(libGroup.Runtime)))) .Where(tuple => tuple.Item2?.AssetPaths != null) .SelectMany( tuple => tuple.Item2.AssetPaths.Where(x => x != null) .Select(path => Tuple.Create(Path.GetFileNameWithoutExtension(path), Tuple.Create(tuple.Item1, tuple.Item2)))) .ToDictionaryIgnoringDuplicateKeys(tuple => tuple.Item1, tuple => tuple.Item2, StringComparer.OrdinalIgnoreCase); internalDiagnosticsMessageSink?.OnMessage( new DiagnosticMessage( $"[DependencyContextAssemblyCache..ctor] Managed assembly map includes: {string.Join(",", _managedAssemblyMap.Keys.Select(k => $"'{k}'").OrderBy(k => k, StringComparer.OrdinalIgnoreCase))}")); #if NETCOREAPP1_0 unmanagedAssemblyCache = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase); unmanagedAssemblyMap = dependencyContext.RuntimeLibraries .Where(lib => lib.NativeLibraryGroups?.Count > 0) .Select(lib => Tuple.Create(lib, lib.NativeLibraryGroups.FirstOrDefault(libGroup => compatibleRuntimes.Contains(libGroup.Runtime)))) .Where(tuple => tuple.Item2?.AssetPaths != null) .SelectMany(tuple => tuple.Item2.AssetPaths.Where(x => x != null).Select(path => Tuple.Create(Path.GetFileName(path), Tuple.Create(tuple.Item1, tuple.Item2)))) .ToDictionaryIgnoringDuplicateKeys(tuple => tuple.Item1, tuple => tuple.Item2, StringComparer.OrdinalIgnoreCase); if (internalDiagnosticsMessageSink != null) { internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[DependencyContextAssemblyCache..ctor] Unmanaged assembly map includes: {string.Join(",", unmanagedAssemblyMap.Keys.Select(k => $"'{k}'").OrderBy(k => k, StringComparer.OrdinalIgnoreCase))}")); } #endif }
public DriverFixture(IMessageSink messageSink) { try { Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); var testWebBrowser = Environment.GetEnvironmentVariable("TestWebBrowser") ?? "Chrome"; messageSink.OnMessage(new DiagnosticMessage($"TestWebBrowser = {testWebBrowser}")); if (testWebBrowser == "Firefox") { var profiles = new FirefoxProfileManager(); var profile = profiles.GetProfile("selenium"); messageSink.OnMessage(new DiagnosticMessage("Profiles:")); foreach (var existingProfile in profiles.ExistingProfiles) { messageSink.OnMessage(new DiagnosticMessage($"Profile = {existingProfile}")); } var options = new FirefoxOptions { Profile = profile }; options.SetLoggingPreference(LogType.Browser, LogLevel.All); Driver = new FirefoxDriver(FirefoxDriverService.CreateDefaultService(Environment.CurrentDirectory), new FirefoxOptions { Profile = profile }, TimeSpan.FromSeconds(30)); } else if (testWebBrowser == "IE") { Driver = new InternetExplorerDriver(Environment.CurrentDirectory); } else if (testWebBrowser == "Chrome") { Driver = new ChromeDriver(Environment.CurrentDirectory); } Driver.Manage().Window.Position = new Point(0, 0); Driver.Manage().Window.Size = new Size(1920, 1080); Driver.Url = Environment.GetEnvironmentVariable("WebUrl") ?? "http://localhost:5000/"; messageSink.OnMessage(new DiagnosticMessage($"WebUrl = {Driver?.Url ?? "NULL"}")); Driver.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(10); Driver.Manage().Timeouts().AsynchronousJavaScript = TimeSpan.FromSeconds(10); Driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(10); Driver.FindElement(By.TagName("app-root")); } catch (Exception ex) { messageSink.OnMessage(new DiagnosticMessage($"Exception while initializing driver: {ex.Message}")); Driver?.Dispose(); } }
void RunTestsInAssembly(IRunContext runContext, IFrameworkHandle frameworkHandle, LoggerHelper logger, IMessageSink reporterMessageHandler, AssemblyRunInfo runInfo) { if (cancelled) { return; } var assembly = new XunitProjectAssembly { AssemblyFilename = runInfo.AssemblyFileName }; var assemblyFileName = runInfo.AssemblyFileName; var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName); var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior; if (DisableAppDomainRequestedInRunContext(runContext.RunSettings.SettingsXml)) { appDomain = AppDomainSupport.Denied; } try { #if PLATFORM_DOTNET // For AppX Apps, use the package location assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName)); #endif var diagnosticSink = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault); using (var controller = new XunitFrontController(appDomain, assemblyFileName: assemblyFileName, configFileName: null, shadowCopy: shadowCopy, diagnosticMessageSink: MessageSinkAdapter.Wrap(diagnosticSink))) { var xunitTestCases = runInfo.TestCases.Select(tc => new { vs = tc, xunit = Deserialize(logger, controller, tc) }) .Where(tc => tc.xunit != null) .ToDictionary(tc => tc.xunit, tc => tc.vs); var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration); executionOptions.SetSynchronousMessageReporting(true); reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); using (var executionSink = new VsExecutionSink(frameworkHandle, logger, xunitTestCases, executionOptions, () => cancelled)) { controller.RunTests(xunitTestCases.Keys.ToList(), executionSink, executionOptions); executionSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, executionSink.ExecutionSummary)); } } } catch (Exception ex) { logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex); } }
/// <summary> /// Initializes a new instance of the <see cref="NetCoreAssemblyDependencyResolver"/> class. /// </summary> /// <param name="assemblyFilePath">The path to the assembly</param> /// <param name="internalDiagnosticsMessageSink">An optional message sink for use with internal diagnostics messages; /// may pass <c>null</c> for no internal diagnostics messages</param> public NetCoreAssemblyDependencyResolver(string assemblyFilePath, IMessageSink internalDiagnosticsMessageSink) { this.internalDiagnosticsMessageSink = internalDiagnosticsMessageSink; var assembly = LoadFromAssemblyPath(assemblyFilePath); assemblyFolder = Path.GetDirectoryName(assemblyFilePath); assemblyResolver = new XunitPackageCompilationAssemblyResolver(); dependencyContext = DependencyContext.Load(assembly); var compatibleRuntimes = default(HashSet <string>); var currentRuntime = RuntimeEnvironment.GetRuntimeIdentifier(); var fallbacks = dependencyContext.RuntimeGraph.FirstOrDefault(x => string.Equals(x.Runtime, currentRuntime, StringComparison.OrdinalIgnoreCase)); if (fallbacks != null) { compatibleRuntimes = new HashSet <string>(fallbacks.Fallbacks, StringComparer.OrdinalIgnoreCase); } else { compatibleRuntimes = new HashSet <string>(StringComparer.OrdinalIgnoreCase); } compatibleRuntimes.Add(currentRuntime); compatibleRuntimes.Add(string.Empty); managedAssemblyCache = new Dictionary <string, Assembly>(StringComparer.OrdinalIgnoreCase); managedAssemblyMap = dependencyContext.RuntimeLibraries .Where(lib => lib.RuntimeAssemblyGroups?.Count > 0) .Select(lib => Tuple.Create(lib, lib.RuntimeAssemblyGroups.FirstOrDefault(libGroup => compatibleRuntimes.Contains(libGroup.Runtime)))) .Where(tuple => tuple.Item2?.AssetPaths != null) .SelectMany(tuple => tuple.Item2.AssetPaths.Where(x => x != null).Select(path => Tuple.Create(Path.GetFileNameWithoutExtension(path), Tuple.Create(tuple.Item1, tuple.Item2)))) .ToDictionaryIgnoringDuplicateKeys(tuple => tuple.Item1, tuple => tuple.Item2, StringComparer.OrdinalIgnoreCase); if (internalDiagnosticsMessageSink != null) { internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[NetCoreAssemblyDependencyResolver..ctor] Managed assembly map includes: {string.Join(",", managedAssemblyMap.Keys.Select(k => $"'{k}'").OrderBy(k => k, StringComparer.OrdinalIgnoreCase))}")); } unmanagedAssemblyCache = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase); unmanagedAssemblyMap = dependencyContext.RuntimeLibraries .Where(lib => lib.NativeLibraryGroups?.Count > 0) .Select(lib => Tuple.Create(lib, lib.NativeLibraryGroups.FirstOrDefault(libGroup => compatibleRuntimes.Contains(libGroup.Runtime)))) .Where(tuple => tuple.Item2?.AssetPaths != null) .SelectMany(tuple => tuple.Item2.AssetPaths.Where(x => x != null).Select(path => Tuple.Create(Path.GetFileName(path), Tuple.Create(tuple.Item1, tuple.Item2)))) .ToDictionaryIgnoringDuplicateKeys(tuple => tuple.Item1, tuple => tuple.Item2, StringComparer.OrdinalIgnoreCase); if (internalDiagnosticsMessageSink != null) { internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[NetCoreAssemblyDependencyResolver..ctor] Unmanaged assembly map includes: {string.Join(",", unmanagedAssemblyMap.Keys.Select(k => $"'{k}'").OrderBy(k => k, StringComparer.OrdinalIgnoreCase))}")); } Default.Resolving += OnResolving; }
/// <summary> /// Runs a retryable test case, handling any wait and retry logic between test runs, reporting statuses out to xunit etc... /// </summary> /// <param name="testCase">The test case to be retried</param> /// <param name="diagnosticMessageSink">The diagnostic message sink to write messages to about retries, waits etc...</param> /// <param name="messageBus">The message bus xunit is listening for statuses to report on</param> /// <param name="cancellationTokenSource">The cancellation token source from xunit</param> /// <param name="fnRunSingle">(async) Lambda to run this test case once (without retries) - takes the blocking message bus and returns the test run result</param> /// <returns>Resulting run summary</returns> public static async Task <RunSummary> RunAsync( IRetryableTestCase testCase, IMessageSink diagnosticMessageSink, IMessageBus messageBus, CancellationTokenSource cancellationTokenSource, Func <IMessageBus, Task <RunSummary> > fnRunSingle) { for (var i = 1; ; i++) { // Prevent messages from the test run from being passed through, as we don't want // a message to mark the test as failed when we're going to retry it using BlockingMessageBus blockingMessageBus = new BlockingMessageBus(messageBus); diagnosticMessageSink.OnMessage(new DiagnosticMessage("Running test \"{0}\" attempt ({1}/{2})", testCase.DisplayName, i, testCase.MaxRetries)); blockingMessageBus.QueueMessage(new DiagnosticMessage("Running test \"{0}\" attempt ({1}/{2})", testCase.DisplayName, i, testCase.MaxRetries)); RunSummary summary = await fnRunSingle(blockingMessageBus); // If we succeeded, or we've reached the max retries return the result if (summary.Failed == 0 || i == testCase.MaxRetries) { // If we have failed (after all retries, log that) if (summary.Failed != 0) { diagnosticMessageSink.OnMessage(new DiagnosticMessage( "Test \"{0}\" has failed and been retried the maximum number of times ({1})", testCase.DisplayName, testCase.MaxRetries)); } blockingMessageBus.Flush(); return(summary); } // Otherwise log that we've had a failed run and will retry diagnosticMessageSink.OnMessage(new DiagnosticMessage( "Test \"{0}\" failed but is set to retry ({1}/{2}) . . .", testCase.DisplayName, i, testCase.MaxRetries)); // If there is a delay between test attempts, apply it now if (testCase.DelayBetweenRetriesMs > 0) { diagnosticMessageSink.OnMessage(new DiagnosticMessage( "Test \"{0}\" attempt ({1}/{2}) delayed by {3}ms. Waiting . . .", testCase.DisplayName, i, testCase.MaxRetries, testCase.DelayBetweenRetriesMs)); // Don't await to prevent thread hopping. // If all of a users test cases in a collection/class are synchronous and expecting to not thread-hop // (because they're making use of thread static/thread local/managed thread ID to share data between tests rather than // a more modern async-friendly mechanism) then if a thread-hop were to happen here we'd get flickering tests. // SpecFlow relies on this as they use the managed thread ID to separate instances of some of their internal classes, which caused // a this problem for xRetry.SpecFlow: https://github.com/JoshKeegan/xRetry/issues/18 Task.Delay(testCase.DelayBetweenRetriesMs, cancellationTokenSource.Token).Wait(); } } }
private void ReturnDiscoveryMessages(IMessageSink sink) { foreach (var testCase in DiscoveryTestCases) { sink.OnMessage(new TestCaseDiscoveryMessage { TestCase = testCase }); } sink.OnMessage(new DiscoveryCompleteMessage()); }
public WinSqlite3TestFramework(IMessageSink messageSink) : base(messageSink) { messageSink.OnMessage(new DiagnosticMessage { Message = "Using custom test framework" }); if (Environment.GetEnvironmentVariable("TEST_WINSQLITE3") != null) { messageSink.OnMessage(new DiagnosticMessage { Message = "Using winsqlite3" }); SqliteEngine.UseWinSqlite3(); } }
/// <inheritdoc/> public override bool OnMessage(IMessageSinkMessage message) { var result = base.OnMessage(message); result = innerMessageSink.OnMessage(message) || result; return(result); }
public ServiceProviderFixture(IMessageSink diagnosticMessageSink) { this.TestId = Guid.NewGuid(); this.Configuration = new ConfigurationBuilder(). AddJsonFile("appsettings.test.json"). AddJsonFile("appsettings.test.Development.json", true). // https://www.jerriepelser.com/blog/aspnet-core-no-more-worries-about-checking-in-secrets/ AddUserSecrets <ServiceProviderFixture>(). AddEnvironmentVariables("CTP_"). Build(); var containerType = Enum.Parse <ContainerType>(Configuration.GetValue("Container", "BuiltIn")); diagnosticMessageSink.OnMessage(new DiagnosticMessage("Use container {0}", containerType.ToString())); switch (containerType) { case ContainerType.BuiltIn: var services = new ServiceCollection(); services.UseCommercetools(Configuration, "Client", TokenFlow.ClientCredentials); services.AddLogging(c => c.AddProvider(new InMemoryLoggerProvider())); this.ServiceProvider = services.BuildServiceProvider(); break; case ContainerType.SimpleInjector: var container = new Container(); container.UseCommercetools(Configuration, "Client", TokenFlow.ClientCredentials); container.Verify(); this.ServiceProvider = container; break; } }
// This method is called by the xUnit test framework classes to run the test case. We will do the // loop here, forwarding on to the implementation in XunitTestCase to do the heavy lifting. We will // continue to re-run the test until the aggregator has an error (meaning that some internal error // condition happened), or the test runs without failure, or we've hit the maximum number of tries. public override async Task <RunSummary> RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) { var runCount = 0; while (true) { // This is really the only tricky bit: we need to capture and delay messages (since those will // contain run status) until we know we've decided to accept the final result; var delayedMessageBus = new DelayedMessageBus(messageBus); var summary = await base.RunAsync(diagnosticMessageSink, delayedMessageBus, constructorArguments, aggregator, cancellationTokenSource); if (aggregator.HasExceptions || summary.Failed == 0 || ++runCount >= maxRetries) { delayedMessageBus.Dispose(); // Sends all the delayed messages return(summary); } diagnosticMessageSink.OnMessage(new DiagnosticMessage("Execution of '{0}' failed (attempt #{1}), retrying...", DisplayName, runCount)); var delayInMs = Math.Min((int)Math.Pow(2, runCount - 1) * this.exponentialBackoffMs, MaxDelayInMs); await Task.Delay(delayInMs); } }
/// <inheritdoc/> protected override void Initialize() { base.Initialize(); var factAttribute = TestMethod.Method.GetCustomAttributes(typeof(FactAttribute)).Single(); var baseDisplayName = factAttribute.GetNamedArgument <string>("DisplayName") ?? BaseDisplayName; DisplayName = TypeUtility.GetDisplayNameWithArguments(TestMethod.Method, baseDisplayName, TestMethodArguments, MethodGenericTypes); SkipReason = factAttribute.GetNamedArgument <string>("Skip"); foreach (var traitAttribute in TestMethod.Method.GetCustomAttributes(typeof(ITraitAttribute)) .Concat(TestMethod.TestClass.Class.GetCustomAttributes(typeof(ITraitAttribute)))) { var discovererAttribute = traitAttribute.GetCustomAttributes(typeof(TraitDiscovererAttribute)).FirstOrDefault(); if (discovererAttribute != null) { var discoverer = ExtensibilityPointFactory.GetTraitDiscoverer(diagnosticMessageSink, discovererAttribute); if (discoverer != null) { foreach (var keyValuePair in discoverer.GetTraits(traitAttribute)) { Traits.Add(keyValuePair.Key, keyValuePair.Value); } } } else { diagnosticMessageSink.OnMessage(new DiagnosticMessage("Trait attribute on '{0}' did not have [TraitDiscoverer]", DisplayName)); } } }
/// <summary> /// Gets the test collection definitions for the given assembly. /// </summary> /// <param name="assemblyInfo">The assembly.</param> /// <param name="diagnosticMessageSink">The message sink used to send diagnostic messages</param> /// <returns>A list of mappings from test collection name to test collection definitions (as <see cref="ITypeInfo"/></returns> public static Dictionary<string, ITypeInfo> GetTestCollectionDefinitions(IAssemblyInfo assemblyInfo, IMessageSink diagnosticMessageSink) { var attributeTypesByName = assemblyInfo.GetTypes(false) .Select(type => new { Type = type, Attribute = type.GetCustomAttributes(typeof(CollectionDefinitionAttribute).AssemblyQualifiedName).FirstOrDefault() }) .Where(list => list.Attribute != null) .GroupBy(list => (string)list.Attribute.GetConstructorArguments().Single(), list => list.Type, StringComparer.OrdinalIgnoreCase); var result = new Dictionary<string, ITypeInfo>(); foreach (var grouping in attributeTypesByName) { var types = grouping.ToList(); result[grouping.Key] = types[0]; if (types.Count > 1) diagnosticMessageSink.OnMessage(new DiagnosticMessage("Multiple test collections declared with name '{0}': {1}", grouping.Key, string.Join(", ", types.Select(type => type.Name)))); } return result; }
public ConditionalTestFramework(IMessageSink messageSink) : base(messageSink) { messageSink.OnMessage(new DiagnosticMessage { Message = "Using " + nameof(ConditionalTestFramework) }); }
/// <inheritdoc /> protected override bool RunTestsOnMethod(IMessageSink messageSink, Type classUnderTest, object[] constructorArguments, MethodInfo methodUnderTest, List<BeforeAfterTestAttribute> beforeAfterAttributes, ExceptionAggregator aggregator, ref decimal executionTime) { try { var testMethod = Reflector.Wrap(methodUnderTest); var dataAttributes = testMethod.GetCustomAttributes(typeof(DataAttribute)); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var args = discovererAttribute.GetConstructorArguments().Cast<string>().ToList(); var discovererType = Reflector.GetType(args[0], args[1]); IDataDiscoverer discoverer = (IDataDiscoverer)Activator.CreateInstance(discovererType); foreach (object[] dataRow in discoverer.GetData(dataAttribute, testMethod)) if (RunTestWithArguments(messageSink, classUnderTest, constructorArguments, methodUnderTest, dataRow, GetDisplayNameWithArguments(DisplayName, dataRow), beforeAfterAttributes, aggregator, ref executionTime)) return true; } return false; } catch (Exception ex) { var cancelled = false; if (!messageSink.OnMessage(new TestStarting { TestCase = this, TestDisplayName = DisplayName })) cancelled = true; else { if (!messageSink.OnMessage(new TestFailed(ex.Unwrap()) { TestCase = this, TestDisplayName = DisplayName })) cancelled = true; } if (!messageSink.OnMessage(new TestFinished { TestCase = this, TestDisplayName = DisplayName })) cancelled = true; return cancelled; } }
/// <inheritdoc /> protected override void RunTestsOnMethod(IMessageSink messageSink, Type classUnderTest, object[] constructorArguments, MethodInfo methodUnderTest, List<BeforeAfterTestAttribute> beforeAfterAttributes, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource, ref decimal executionTime) { try { var testMethod = Reflector.Wrap(methodUnderTest); var dataAttributes = testMethod.GetCustomAttributes(typeof(DataAttribute)); foreach (var dataAttribute in dataAttributes) { var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First(); var args = discovererAttribute.GetConstructorArguments().Cast<string>().ToList(); var discovererType = Reflector.GetType(args[1], args[0]); IDataDiscoverer discoverer = (IDataDiscoverer)Activator.CreateInstance(discovererType); foreach (object[] dataRow in discoverer.GetData(dataAttribute, testMethod)) { RunTestWithArguments(messageSink, classUnderTest, constructorArguments, methodUnderTest, dataRow, GetDisplayNameWithArguments(DisplayName, dataRow), beforeAfterAttributes, aggregator, cancellationTokenSource, ref executionTime); if (cancellationTokenSource.IsCancellationRequested) return; } } } catch (Exception ex) { if (!messageSink.OnMessage(new TestStarting(this, DisplayName))) cancellationTokenSource.Cancel(); else { if (!messageSink.OnMessage(new TestFailed(this, DisplayName, executionTime, ex.Unwrap()))) cancellationTokenSource.Cancel(); } if (!messageSink.OnMessage(new TestFinished(this, DisplayName, executionTime))) cancellationTokenSource.Cancel(); } }
/// <inheritdoc/> public void Find(bool includeSourceInformation, IMessageSink messageSink) { Guard.ArgumentNotNull("messageSink", messageSink); ThreadPool.QueueUserWorkItem(_ => { foreach (var type in assemblyInfo.GetTypes(includePrivateTypes: false)) if (!FindImpl(type, includeSourceInformation, messageSink)) break; messageSink.OnMessage(new DiscoveryCompleteMessage()); }); }
/// <inheritdoc/> protected override void RunTests(IMessageSink messageSink, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) { if (!messageSink.OnMessage(new TestStarting(this, DisplayName))) cancellationTokenSource.Cancel(); else { try { lambda(); if (!messageSink.OnMessage(new TestPassed(this, DisplayName, 0))) cancellationTokenSource.Cancel(); } catch (Exception ex) { if (!messageSink.OnMessage(new TestFailed(this, DisplayName, 0, ex))) cancellationTokenSource.Cancel(); } } if (!messageSink.OnMessage(new TestFinished(this, DisplayName, 0))) cancellationTokenSource.Cancel(); }
/// <inheritdoc/> public void Find(string typeName, bool includeSourceInformation, IMessageSink messageSink) { Guard.ArgumentNotNullOrEmpty("typeName", typeName); Guard.ArgumentNotNull("messageSink", messageSink); ThreadPool.QueueUserWorkItem(_ => { ITypeInfo typeInfo = assemblyInfo.GetType(typeName); if (typeInfo != null) FindImpl(typeInfo, includeSourceInformation, messageSink); messageSink.OnMessage(new DiscoveryCompleteMessage()); }); }
/// <inheritdoc/> public async void Run(IEnumerable<ITestCase> testCases, IMessageSink messageSink) { var cancellationTokenSource = new CancellationTokenSource(); var totalSummary = new RunSummary(); string currentDirectory = Directory.GetCurrentDirectory(); try { Directory.SetCurrentDirectory(Path.GetDirectoryName(assemblyInfo.AssemblyPath)); if (messageSink.OnMessage(new TestAssemblyStarting(assemblyFileName, AppDomain.CurrentDomain.SetupInformation.ConfigurationFile, DateTime.Now, String.Format("{0}-bit .NET {1}", IntPtr.Size * 8, Environment.Version), XunitTestFrameworkDiscoverer.DisplayName))) { // TODO: Contract for Run() states that null "testCases" means "run everything". var tasks = testCases.Cast<XunitTestCase>() .GroupBy(tc => tc.TestCollection) .Select(collectionGroup => Task.Run(() => RunTestCollection(messageSink, collectionGroup.Key, collectionGroup, cancellationTokenSource))) .ToArray(); var summaries = await Task.WhenAll(tasks); totalSummary.Time = summaries.Sum(s => s.Time); totalSummary.Total = summaries.Sum(s => s.Total); totalSummary.Failed = summaries.Sum(s => s.Failed); totalSummary.Skipped = summaries.Sum(s => s.Skipped); } messageSink.OnMessage(new TestAssemblyFinished(assemblyInfo, totalSummary.Time, totalSummary.Total, totalSummary.Failed, totalSummary.Skipped)); } finally { Directory.SetCurrentDirectory(currentDirectory); } }
static ITestFramework CreateInnerTestFramework(Type testFrameworkType, IMessageSink diagnosticMessageSink) { try { var ctorWithSink = testFrameworkType.GetTypeInfo().DeclaredConstructors .FirstOrDefault(ctor => { var paramInfos = ctor.GetParameters(); return paramInfos.Length == 1 && paramInfos[0].ParameterType == typeof(IMessageSink); }); if (ctorWithSink != null) return (ITestFramework)ctorWithSink.Invoke(new object[] { diagnosticMessageSink }); return (ITestFramework)Activator.CreateInstance(testFrameworkType); } catch (Exception ex) { diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Exception thrown during test framework construction: {ex.Unwrap()}")); return new XunitTestFramework(diagnosticMessageSink); } }
// This method is called by the xUnit test framework classes to run the test case. We will do the // loop here, forwarding on to the implementation in XunitTestCase to do the heavy lifting. We will // continue to re-run the test until the aggregator has an error (meaning that some internal error // condition happened), or the test runs without failure, or we've hit the maximum number of tries. public override async Task<RunSummary> RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) { var runCount = 0; while (true) { // This is really the only tricky bit: we need to capture and delay messages (since those will // contain run status) until we know we've decided to accept the final result; var delayedMessageBus = new DelayedMessageBus(messageBus); var summary = await base.RunAsync(diagnosticMessageSink, delayedMessageBus, constructorArguments, aggregator, cancellationTokenSource); if (aggregator.HasExceptions || summary.Failed == 0 || ++runCount >= maxRetries) { delayedMessageBus.Dispose(); // Sends all the delayed messages return summary; } diagnosticMessageSink.OnMessage(new DiagnosticMessage("Execution of '{0}' failed (attempt #{1}), retrying...", DisplayName, runCount)); } }
RunSummary RunTestClass(ITestCollection testCollection, string className, IList<Xunit1TestCase> testCases, IMessageSink messageSink) { var handler = new TestClassCallbackHandler(testCases, messageSink); var results = handler.TestClassResults; results.Continue = messageSink.OnMessage(new TestClassStarting(testCollection, className)); if (results.Continue) { try { var methodNames = testCases.Select(tc => tc.Method.Name).ToList(); executor.RunTests(className, methodNames, handler); handler.LastNodeArrived.WaitOne(); } catch (Exception ex) { var failureInformation = Xunit1ExceptionUtility.ConvertToFailureInformation(ex); results.Continue = messageSink.OnMessage(new ErrorMessage(failureInformation.ExceptionTypes, failureInformation.Messages, failureInformation.StackTraces, failureInformation.ExceptionParentIndices)) && results.Continue; } } results.Continue = messageSink.OnMessage(new TestClassFinished(testCollection, className, results.Time, results.Total, results.Failed, results.Skipped)) && results.Continue; return results; }
public override bool Execute() { RemotingUtility.CleanUpRegisteredChannels(); if (TeamCity) { Log.LogError("The 'TeamCity' property is deprecated. Please set the 'Reporter' property to 'teamcity' instead."); return false; } if (Verbose) { Log.LogError("The 'Verbose' property is deprecated. Please set the 'Reporter' property to 'verbose' instead."); return false; } XElement assembliesElement = null; var environment = string.Format("{0}-bit .NET {1}", IntPtr.Size * 8, Environment.Version); if (NeedsXml) assembliesElement = new XElement("assemblies"); switch (MaxParallelThreads) { case null: case "default": break; case "unlimited": maxThreadCount = 0; break; default: int threadValue; if (!int.TryParse(MaxParallelThreads, out threadValue) || threadValue < 0) { Log.LogError("MaxParallelThreads value '{0}' is invalid: must be 'default', 'unlimited', or a positive number", MaxParallelThreads); return false; } maxThreadCount = threadValue; break; } var originalWorkingFolder = Directory.GetCurrentDirectory(); using (AssemblyHelper.SubscribeResolve()) { var reporters = GetAvailableRunnerReporters(); var reporter = reporters.FirstOrDefault(r => r.IsEnvironmentallyEnabled); if (reporter == null && !string.IsNullOrWhiteSpace(Reporter)) { reporter = reporters.FirstOrDefault(r => string.Equals(r.RunnerSwitch, Reporter, StringComparison.OrdinalIgnoreCase)); if (reporter == null) { var switchableReporters = reporters.Where(r => !string.IsNullOrWhiteSpace(r.RunnerSwitch)).Select(r => r.RunnerSwitch.ToLowerInvariant()).OrderBy(x => x).ToList(); if (switchableReporters.Count == 0) Log.LogError("Reporter value '{0}' is invalid. There are no available reporters.", Reporter); else Log.LogError("Reporter value '{0}' is invalid. Available reporters: {1}", Reporter, string.Join(", ", switchableReporters)); return false; } } if (reporter == null) reporter = new DefaultRunnerReporter(); logger = new MSBuildLogger(Log); reporterMessageHandler = reporter.CreateMessageHandler(logger); if (!NoLogo) Log.LogMessage(MessageImportance.High, "xUnit.net MSBuild Runner ({0})", environment); var project = new XunitProject(); foreach (var assembly in Assemblies) { var assemblyFileName = assembly.GetMetadata("FullPath"); var configFileName = assembly.GetMetadata("ConfigFile"); if (configFileName != null && configFileName.Length == 0) configFileName = null; project.Add(new XunitProjectAssembly { AssemblyFilename = assemblyFileName, ConfigFilename = configFileName, ShadowCopy = ShadowCopy }); } if (WorkingFolder != null) Directory.SetCurrentDirectory(WorkingFolder); var clockTime = Stopwatch.StartNew(); if (!parallelizeAssemblies.HasValue) parallelizeAssemblies = project.All(assembly => assembly.Configuration.ParallelizeAssemblyOrDefault); if (parallelizeAssemblies.GetValueOrDefault()) { var tasks = project.Assemblies.Select(assembly => Task.Run(() => ExecuteAssembly(assembly))); var results = Task.WhenAll(tasks).GetAwaiter().GetResult(); foreach (var assemblyElement in results.Where(result => result != null)) assembliesElement.Add(assemblyElement); } else { foreach (var assembly in project.Assemblies) { var assemblyElement = ExecuteAssembly(assembly); if (assemblyElement != null) assembliesElement.Add(assemblyElement); } } clockTime.Stop(); if (completionMessages.Count > 0) reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, completionMessages.OrderBy(kvp => kvp.Key).ToList())); } Directory.SetCurrentDirectory(WorkingFolder ?? originalWorkingFolder); if (NeedsXml) { if (Xml != null) assembliesElement.Save(Xml.GetMetadata("FullPath")); if (XmlV1 != null) Transform("xUnit1.xslt", assembliesElement, XmlV1); if (Html != null) Transform("HTML.xslt", assembliesElement, Html); if (NUnit != null) Transform("NUnitXml.xslt", assembliesElement, NUnit); } return ExitCode == 0; }
/// <inheritdoc/> public void Find(bool includeSourceInformation, IMessageSink messageSink) { Guard.ArgumentNotNull("messageSink", messageSink); ThreadPool.QueueUserWorkItem(_ => { foreach (var type in assemblyInfo.GetTypes(includePrivateTypes: false)) if (!FindImpl(type, includeSourceInformation, messageSink)) break; var warnings = messageAggregator.GetAndClear<EnvironmentalWarning>().Select(w => w.Message).ToList(); messageSink.OnMessage(new DiscoveryCompleteMessage(warnings)); }); }
/// <summary> /// Runs a single test for a given test method. /// </summary> /// <param name="messageSink">The message sink to send results to.</param> /// <param name="classUnderTest">The class under test.</param> /// <param name="constructorArguments">The arguments to pass to the constructor.</param> /// <param name="methodUnderTest">The method under test.</param> /// <param name="testMethodArguments">The arguments to pass to the test method.</param> /// <param name="displayName">The display name for the test.</param> /// <param name="beforeAfterAttributes">The <see cref="BeforeAfterTestAttribute"/> instances attached to the test.</param> /// <param name="aggregator">The error aggregator to use for catching exception.</param> /// <param name="executionTime">The time spent executing the tests.</param> protected bool RunTestWithArguments(IMessageSink messageSink, Type classUnderTest, object[] constructorArguments, MethodInfo methodUnderTest, object[] testMethodArguments, string displayName, List<BeforeAfterTestAttribute> beforeAfterAttributes, ExceptionAggregator aggregator, ref decimal executionTime) { bool cancelled = false; if (!messageSink.OnMessage(new TestStarting { TestCase = this, TestDisplayName = displayName })) cancelled = true; else { if (!String.IsNullOrEmpty(SkipReason)) { if (!messageSink.OnMessage(new TestSkipped { TestCase = this, TestDisplayName = DisplayName, Reason = SkipReason })) cancelled = true; } else { var beforeAttributesRun = new Stack<BeforeAfterTestAttribute>(); var stopwatch = Stopwatch.StartNew(); if (!aggregator.HasExceptions) { aggregator.Run(() => { object testClass = null; if (!methodUnderTest.IsStatic) { if (!messageSink.OnMessage(new TestClassConstructionStarting { TestCase = this, TestDisplayName = displayName })) cancelled = true; try { if (!cancelled) testClass = Activator.CreateInstance(classUnderTest, constructorArguments); } finally { if (!messageSink.OnMessage(new TestClassConstructionFinished { TestCase = this, TestDisplayName = displayName })) cancelled = true; } } if (!cancelled) { aggregator.Run(() => { foreach (var beforeAfterAttribute in beforeAfterAttributes) { if (!messageSink.OnMessage(new BeforeTestStarting { TestCase = this, TestDisplayName = displayName, AttributeName = beforeAfterAttribute.GetType().Name })) cancelled = true; else { try { beforeAfterAttribute.Before(methodUnderTest); beforeAttributesRun.Push(beforeAfterAttribute); } finally { if (!messageSink.OnMessage(new BeforeTestFinished { TestCase = this, TestDisplayName = displayName, AttributeName = beforeAfterAttribute.GetType().Name })) cancelled = true; } } if (cancelled) return; } if (!cancelled) { var parameterTypes = methodUnderTest.GetParameters().Select(p => p.ParameterType).ToArray(); aggregator.Run(() => { var result = methodUnderTest.Invoke(testClass, ConvertArguments(testMethodArguments ?? EmptyArray, parameterTypes)); var task = result as Task; if (task != null) task.GetAwaiter().GetResult(); }); } }); foreach (var beforeAfterAttribute in beforeAttributesRun) { if (!messageSink.OnMessage(new AfterTestStarting { TestCase = this, TestDisplayName = displayName, AttributeName = beforeAfterAttribute.GetType().Name })) cancelled = true; aggregator.Run(() => beforeAfterAttribute.After(methodUnderTest)); if (!messageSink.OnMessage(new AfterTestFinished { TestCase = this, TestDisplayName = displayName, AttributeName = beforeAfterAttribute.GetType().Name })) cancelled = true; } } aggregator.Run(() => { IDisposable disposable = testClass as IDisposable; if (disposable != null) { if (!messageSink.OnMessage(new TestClassDisposeStarting { TestCase = this, TestDisplayName = displayName })) cancelled = true; try { disposable.Dispose(); } finally { if (!messageSink.OnMessage(new TestClassDisposeFinished { TestCase = this, TestDisplayName = displayName })) cancelled = true; } } }); }); } stopwatch.Stop(); if (!cancelled) { executionTime = (decimal)stopwatch.Elapsed.TotalSeconds; var exception = aggregator.ToException(); var testResult = exception == null ? (TestResultMessage)new TestPassed() : new TestFailed(exception); testResult.TestCase = this; testResult.TestDisplayName = displayName; testResult.ExecutionTime = executionTime; if (!messageSink.OnMessage(testResult)) cancelled = true; } } } if (!messageSink.OnMessage(new TestFinished { TestCase = this, TestDisplayName = displayName, ExecutionTime = executionTime })) cancelled = true; return cancelled; }
private static bool RunTestClass(IMessageSink messageSink, IGrouping<ITypeInfo, XunitTestCase> group, RunSummary classSummary) { bool cancelled = false; var aggregator = new ExceptionAggregator(); Type testClassType = ((IReflectionTypeInfo)group.Key).Type; Dictionary<Type, object> fixtureMappings = new Dictionary<Type, object>(); List<object> constructorArguments = new List<object>(); // TODO: Read class fixtures from test collection foreach (var iface in testClassType.GetInterfaces().Where(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IClassFixture<>))) { Type fixtureType = iface.GetGenericArguments().Single(); object fixture = null; aggregator.Run(() => fixture = Activator.CreateInstance(fixtureType)); fixtureMappings.Add(fixtureType, fixture); } var ctors = testClassType.GetConstructors(); if (ctors.Length != 1) { aggregator.Add(new TestClassException("A test class may only define a single public constructor.")); } else { var ctor = ctors.Single(); List<string> unusedArguments = new List<string>(); foreach (var paramInfo in ctor.GetParameters()) { object fixture; if (fixtureMappings.TryGetValue(paramInfo.ParameterType, out fixture)) constructorArguments.Add(fixture); else unusedArguments.Add(paramInfo.ParameterType.Name + " " + paramInfo.Name); } if (unusedArguments.Count > 0) aggregator.Add(new TestClassException("The following constructor arguments did not have matching fixture data: " + String.Join(", ", unusedArguments))); } var methodGroups = group.GroupBy(tc => tc.Method); foreach (var method in methodGroups) { if (!messageSink.OnMessage(new TestMethodStarting { ClassName = group.Key.Name, MethodName = method.Key.Name })) cancelled = true; else cancelled = RunTestMethod(messageSink, constructorArguments.ToArray(), method, classSummary, aggregator); if (!messageSink.OnMessage(new TestMethodFinished { ClassName = group.Key.Name, MethodName = method.Key.Name })) cancelled = true; if (cancelled) break; } foreach (var fixture in fixtureMappings.Values.OfType<IDisposable>()) { try { fixture.Dispose(); } catch (Exception ex) { if (!messageSink.OnMessage(new ErrorMessage(ex.Unwrap()))) cancelled = true; } } return cancelled; }
private void ReturnDiscoveryMessages(IMessageSink sink) { foreach (var testCase in DiscoveryTestCases) sink.OnMessage(new TestCaseDiscoveryMessage { TestCase = testCase }); sink.OnMessage(new DiscoveryCompleteMessage()); }
/// <summary> /// Core implementation to discover unit tests in a given test class. /// </summary> /// <param name="type">The test class.</param> /// <param name="includeSourceInformation">Set to <c>true</c> to attempt to include source information.</param> /// <param name="messageSink">The message sink to send discovery messages to.</param> /// <returns>Returns <c>true</c> if discovery should continue; <c>false</c> otherwise.</returns> protected virtual bool FindImpl(ITypeInfo type, bool includeSourceInformation, IMessageSink messageSink) { string currentDirectory = Directory.GetCurrentDirectory(); var testCollection = TestCollectionFactory.Get(type); try { if (!String.IsNullOrEmpty(assemblyInfo.AssemblyPath)) Directory.SetCurrentDirectory(Path.GetDirectoryName(assemblyInfo.AssemblyPath)); foreach (IMethodInfo method in type.GetMethods(includePrivateMethods: true)) { IAttributeInfo factAttribute = method.GetCustomAttributes(typeof(FactAttribute)).FirstOrDefault(); if (factAttribute != null) { IAttributeInfo discovererAttribute = factAttribute.GetCustomAttributes(typeof(XunitDiscovererAttribute)).FirstOrDefault(); if (discovererAttribute != null) { var args = discovererAttribute.GetConstructorArguments().Cast<string>().ToList(); var discovererType = Reflector.GetType(args[1], args[0]); if (discovererType != null) { IXunitDiscoverer discoverer = (IXunitDiscoverer)Activator.CreateInstance(discovererType); foreach (XunitTestCase testCase in discoverer.Discover(testCollection, assemblyInfo, type, method, factAttribute)) if (!messageSink.OnMessage(new TestCaseDiscoveryMessage(UpdateTestCaseWithSourceInfo(testCase, includeSourceInformation)))) return false; } // TODO: Figure out a way to report back an error when discovererType is not available // TODO: What if the discovererType can't be created or cast to IXunitDiscoverer? // TODO: Performance optimization: cache instances of the discoverer type } } } return true; } finally { Directory.SetCurrentDirectory(currentDirectory); } }
void Find(Predicate<ITestCaseDiscoveryMessage> filter, bool includeSourceInformation, IMessageSink messageSink) { try { XmlNode assemblyXml = null; using (var handler = new XmlNodeCallbackHandler(xml => { assemblyXml = xml; return true; })) executor.EnumerateTests(handler); foreach (XmlNode method in assemblyXml.SelectNodes("//method")) { var testCase = method.ToTestCase(assemblyFileName); if (testCase != null) { if (includeSourceInformation) testCase.SourceInformation = sourceInformationProvider.GetSourceInformation(testCase); testCase.TestCollection = testCollection; var message = new TestCaseDiscoveryMessage(testCase); if (filter(message)) messageSink.OnMessage(message); } } } finally { messageSink.OnMessage(new DiscoveryCompleteMessage(new string[0])); } }
private RunSummary RunTestCollection(IMessageSink messageSink, ITestCollection collection, IEnumerable<XunitTestCase> testCases, CancellationTokenSource cancellationTokenSource) { var collectionSummary = new RunSummary(); var collectionFixtureMappings = new Dictionary<Type, object>(); var aggregator = new ExceptionAggregator(); if (collection.CollectionDefinition != null) { var declarationType = ((IReflectionTypeInfo)collection.CollectionDefinition).Type; foreach (var interfaceType in declarationType.GetInterfaces().Where(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(ICollectionFixture<>))) CreateFixture(interfaceType, aggregator, collectionFixtureMappings); } if (messageSink.OnMessage(new TestCollectionStarting(collection))) { foreach (var testCasesByClass in testCases.GroupBy(tc => tc.Class)) { var classSummary = new RunSummary(); if (!messageSink.OnMessage(new TestClassStarting(collection, testCasesByClass.Key.Name))) cancellationTokenSource.Cancel(); else { RunTestClass(messageSink, collection, collectionFixtureMappings, (IReflectionTypeInfo)testCasesByClass.Key, testCasesByClass, classSummary, aggregator, cancellationTokenSource); collectionSummary.Aggregate(classSummary); } if (!messageSink.OnMessage(new TestClassFinished(collection, testCasesByClass.Key.Name, classSummary.Time, classSummary.Total, classSummary.Failed, classSummary.Skipped))) cancellationTokenSource.Cancel(); if (cancellationTokenSource.IsCancellationRequested) break; } } foreach (var fixture in collectionFixtureMappings.Values.OfType<IDisposable>()) { try { fixture.Dispose(); } catch (Exception ex) { if (!messageSink.OnMessage(new ErrorMessage(ex.Unwrap()))) cancellationTokenSource.Cancel(); } } messageSink.OnMessage(new TestCollectionFinished(collection, collectionSummary.Time, collectionSummary.Total, collectionSummary.Failed, collectionSummary.Skipped)); return collectionSummary; }
protected static bool OnMessage(IMessageSink messageSink, IMessageSinkMessage message) { var result = messageSink.OnMessage(message); RemotingServices.Disconnect((MarshalByRefObject)message); return result; }
/// <summary> /// Runs a single test for a given test method. /// </summary> /// <param name="messageSink">The message sink to send results to.</param> /// <param name="classUnderTest">The class under test.</param> /// <param name="constructorArguments">The arguments to pass to the constructor.</param> /// <param name="methodUnderTest">The method under test.</param> /// <param name="testMethodArguments">The arguments to pass to the test method.</param> /// <param name="displayName">The display name for the test.</param> /// <param name="beforeAfterAttributes">The <see cref="BeforeAfterTestAttribute"/> instances attached to the test.</param> /// <param name="parentAggregator">The parent aggregator that contains the exceptions up to this point.</param> /// <param name="cancellationTokenSource">The cancellation token source that indicates whether cancellation has been requested.</param> /// <param name="executionTime">The time spent executing the tests.</param> protected void RunTestWithArguments(IMessageSink messageSink, Type classUnderTest, object[] constructorArguments, MethodInfo methodUnderTest, object[] testMethodArguments, string displayName, List<BeforeAfterTestAttribute> beforeAfterAttributes, ExceptionAggregator parentAggregator, CancellationTokenSource cancellationTokenSource, ref decimal executionTime) { var aggregator = new ExceptionAggregator(parentAggregator); if (!messageSink.OnMessage(new TestStarting(this, displayName))) cancellationTokenSource.Cancel(); else { if (!String.IsNullOrEmpty(SkipReason)) { if (!messageSink.OnMessage(new TestSkipped(this, displayName, SkipReason))) cancellationTokenSource.Cancel(); } else { var beforeAttributesRun = new Stack<BeforeAfterTestAttribute>(); var stopwatch = Stopwatch.StartNew(); if (!aggregator.HasExceptions) aggregator.Run(() => { object testClass = null; if (!methodUnderTest.IsStatic) { if (!messageSink.OnMessage(new TestClassConstructionStarting(this, displayName))) cancellationTokenSource.Cancel(); try { if (!cancellationTokenSource.IsCancellationRequested) testClass = Activator.CreateInstance(classUnderTest, constructorArguments); } finally { if (!messageSink.OnMessage(new TestClassConstructionFinished(this, displayName))) cancellationTokenSource.Cancel(); } } if (!cancellationTokenSource.IsCancellationRequested) { aggregator.Run(() => { foreach (var beforeAfterAttribute in beforeAfterAttributes) { var attributeName = beforeAfterAttribute.GetType().Name; if (!messageSink.OnMessage(new BeforeTestStarting(this, displayName, attributeName))) cancellationTokenSource.Cancel(); else { try { beforeAfterAttribute.Before(methodUnderTest); beforeAttributesRun.Push(beforeAfterAttribute); } finally { if (!messageSink.OnMessage(new BeforeTestFinished(this, displayName, attributeName))) cancellationTokenSource.Cancel(); } } if (cancellationTokenSource.IsCancellationRequested) return; } if (!cancellationTokenSource.IsCancellationRequested) { var parameterTypes = methodUnderTest.GetParameters().Select(p => p.ParameterType).ToArray(); var oldSyncContext = SynchronizationContext.Current; try { var asyncSyncContext = new AsyncTestSyncContext(); SetSynchronizationContext(asyncSyncContext); aggregator.Run(() => { var result = methodUnderTest.Invoke(testClass, ConvertArguments(testMethodArguments ?? EmptyArray, parameterTypes)); var task = result as Task; if (task != null) task.GetAwaiter().GetResult(); else { var ex = asyncSyncContext.WaitForCompletion(); if (ex != null) aggregator.Add(ex); } }); } finally { SetSynchronizationContext(oldSyncContext); } } }); foreach (var beforeAfterAttribute in beforeAttributesRun) { var attributeName = beforeAfterAttribute.GetType().Name; if (!messageSink.OnMessage(new AfterTestStarting(this, displayName, attributeName))) cancellationTokenSource.Cancel(); aggregator.Run(() => beforeAfterAttribute.After(methodUnderTest)); if (!messageSink.OnMessage(new AfterTestFinished(this, displayName, attributeName))) cancellationTokenSource.Cancel(); } } aggregator.Run(() => { IDisposable disposable = testClass as IDisposable; if (disposable != null) { if (!messageSink.OnMessage(new TestClassDisposeStarting(this, displayName))) cancellationTokenSource.Cancel(); try { disposable.Dispose(); } finally { if (!messageSink.OnMessage(new TestClassDisposeFinished(this, displayName))) cancellationTokenSource.Cancel(); } } }); }); stopwatch.Stop(); if (!cancellationTokenSource.IsCancellationRequested) { executionTime = (decimal)stopwatch.Elapsed.TotalSeconds; var exception = aggregator.ToException(); var testResult = exception == null ? (TestResultMessage)new TestPassed(this, displayName, executionTime) : new TestFailed(this, displayName, executionTime, exception); if (!messageSink.OnMessage(testResult)) cancellationTokenSource.Cancel(); } } } if (!messageSink.OnMessage(new TestFinished(this, displayName, executionTime))) cancellationTokenSource.Cancel(); }
/// <summary> /// Executes the test case, returning 0 or more result messages through the message sink. /// </summary> /// <param name="messageSink">The message sink to report results to.</param> /// <param name="constructorArguments">The arguments to pass to the constructor.</param> /// <param name="aggregator">The error aggregator to use for catching exception.</param> /// <param name="cancellationTokenSource">The cancellation token source that indicates whether cancellation has been requested.</param> public virtual void Run(IMessageSink messageSink, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) { int totalFailed = 0; int totalRun = 0; int totalSkipped = 0; decimal executionTime = 0M; if (!messageSink.OnMessage(new TestCaseStarting(this))) cancellationTokenSource.Cancel(); else { var delegatingSink = new DelegatingMessageSink(messageSink, msg => { if (msg is ITestResultMessage) { totalRun++; executionTime += ((ITestResultMessage)msg).ExecutionTime; } if (msg is ITestFailed) totalFailed++; if (msg is ITestSkipped) totalSkipped++; }); RunTests(delegatingSink, constructorArguments, aggregator, cancellationTokenSource); } if (!messageSink.OnMessage(new TestCaseFinished(this, executionTime, totalRun, totalFailed, totalSkipped))) cancellationTokenSource.Cancel(); }
static Type GetTestFrameworkType(IAssemblyInfo testAssembly, IMessageSink diagnosticMessageSink) { try { var testFrameworkAttr = testAssembly.GetCustomAttributes(typeof(ITestFrameworkAttribute)).FirstOrDefault(); if (testFrameworkAttr != null) { var discovererAttr = testFrameworkAttr.GetCustomAttributes(typeof(TestFrameworkDiscovererAttribute)).FirstOrDefault(); if (discovererAttr != null) { var discoverer = ExtensibilityPointFactory.GetTestFrameworkTypeDiscoverer(diagnosticMessageSink, discovererAttr); if (discoverer != null) return discoverer.GetTestFrameworkType(testFrameworkAttr); var ctorArgs = discovererAttr.GetConstructorArguments().ToArray(); diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Unable to create custom test framework discoverer type '{ctorArgs[1]}, {ctorArgs[0]}'")); } else { diagnosticMessageSink.OnMessage(new DiagnosticMessage("Assembly-level test framework attribute was not decorated with [TestFrameworkDiscoverer]")); } } } catch (Exception ex) { diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Exception thrown during test framework discoverer construction: {ex.Unwrap()}")); } return typeof(XunitTestFramework); }
RunSummary RunTestCollection(ITestCollection testCollection, IEnumerable<Xunit1TestCase> testCases, IMessageSink messageSink) { var results = new RunSummary(); results.Continue = messageSink.OnMessage(new TestCollectionStarting(testCollection)); if (results.Continue) foreach (var testClassGroup in testCases.GroupBy(tc => tc.Class.Name)) { var classResults = RunTestClass(testCollection, testClassGroup.Key, testClassGroup.ToList(), messageSink); results.Aggregate(classResults); if (!classResults.Continue) break; } results.Continue = messageSink.OnMessage(new TestCollectionFinished(testCollection, results.Time, results.Total, results.Failed, results.Skipped)) && results.Continue; return results; }
RunSummary RunTestClass(ITestCollection testCollection, string className, IList<Xunit1TestCase> testCases, IMessageSink messageSink) { var handler = new TestClassCallbackHandler(testCases, messageSink); var results = handler.TestClassResults; results.Continue = messageSink.OnMessage(new TestClassStarting(testCollection, className)); if (results.Continue) { try { var methodNames = testCases.Select(tc => tc.Method.Name).ToList(); executor.RunTests(className, methodNames, handler); handler.LastNodeArrived.WaitOne(); } catch (Exception ex) { var stackTrace = ex.StackTrace; var rethrowIndex = stackTrace.IndexOf("$$RethrowMarker$$"); if (rethrowIndex > -1) stackTrace = stackTrace.Substring(0, rethrowIndex); results.Continue = messageSink.OnMessage(new ErrorMessage(ex.GetType().FullName, ex.Message, stackTrace)) && results.Continue; } } results.Continue = messageSink.OnMessage(new TestClassFinished(testCollection, className, results.Time, results.Total, results.Failed, results.Skipped)) && results.Continue; return results; }
/// <inheritdoc/> public void Find(string typeName, bool includeSourceInformation, IMessageSink messageSink) { Guard.ArgumentNotNullOrEmpty("typeName", typeName); Guard.ArgumentNotNull("messageSink", messageSink); ThreadPool.QueueUserWorkItem(_ => { ITypeInfo typeInfo = assemblyInfo.GetType(typeName); if (typeInfo != null) FindImpl(typeInfo, includeSourceInformation, messageSink); var warnings = messageAggregator.GetAndClear<EnvironmentalWarning>().Select(w => w.Message).ToList(); messageSink.OnMessage(new DiscoveryCompleteMessage(warnings)); }); }
/// <inheritdoc/> public void Run(IEnumerable<ITestCase> testCases, IMessageSink messageSink) { var results = new RunSummary(); var environment = String.Format("{0}-bit .NET {1}", IntPtr.Size * 8, Environment.Version); // TODO: Contract for Run() states that null "testCases" means "run everything". if (messageSink.OnMessage(new TestAssemblyStarting(assemblyFileName, configFileName, DateTime.Now, environment, TestFrameworkDisplayName))) foreach (var testCollectionGroup in testCases.Cast<Xunit1TestCase>().GroupBy(tc => tc.TestCollection)) { var collectionResults = RunTestCollection(testCollectionGroup.Key, testCollectionGroup, messageSink); results.Aggregate(collectionResults); if (!collectionResults.Continue) break; } messageSink.OnMessage(new TestAssemblyFinished(new Xunit1AssemblyInfo(assemblyFileName), results.Time, results.Total, results.Failed, results.Skipped)); }
private static void RunTestClass(IMessageSink messageSink, ITestCollection collection, Dictionary<Type, object> collectionFixtureMappings, IReflectionTypeInfo testClass, IEnumerable<XunitTestCase> testCases, RunSummary classSummary, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) { var testClassType = testClass.Type; var fixtureMappings = new Dictionary<Type, object>(); var constructorArguments = new List<object>(); if (testClassType.GetInterfaces().Any(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(ICollectionFixture<>))) aggregator.Add(new TestClassException("A test class may not be decorated with ICollectionFixture<> (decorate the test collection class instead).")); foreach (var interfaceType in testClassType.GetInterfaces().Where(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IClassFixture<>))) CreateFixture(interfaceType, aggregator, fixtureMappings); if (collection.CollectionDefinition != null) { var declarationType = ((IReflectionTypeInfo)collection.CollectionDefinition).Type; foreach (var interfaceType in declarationType.GetInterfaces().Where(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IClassFixture<>))) CreateFixture(interfaceType, aggregator, fixtureMappings); } var isStaticClass = testClassType.IsAbstract && testClassType.IsSealed; if (!isStaticClass) { var ctors = testClassType.GetConstructors(); if (ctors.Length != 1) { aggregator.Add(new TestClassException("A test class may only define a single public constructor.")); } else { var ctor = ctors.Single(); var unusedArguments = new List<string>(); foreach (var paramInfo in ctor.GetParameters()) { object fixture; if (fixtureMappings.TryGetValue(paramInfo.ParameterType, out fixture) || collectionFixtureMappings.TryGetValue(paramInfo.ParameterType, out fixture)) constructorArguments.Add(fixture); else unusedArguments.Add(paramInfo.ParameterType.Name + " " + paramInfo.Name); } if (unusedArguments.Count > 0) aggregator.Add(new TestClassException("The following constructor arguments did not have matching fixture data: " + String.Join(", ", unusedArguments))); } } var methodGroups = testCases.GroupBy(tc => tc.Method); foreach (var method in methodGroups) { if (!messageSink.OnMessage(new TestMethodStarting(collection, testClass.Name, method.Key.Name))) cancellationTokenSource.Cancel(); else RunTestMethod(messageSink, constructorArguments.ToArray(), method, classSummary, aggregator, cancellationTokenSource); if (!messageSink.OnMessage(new TestMethodFinished(collection, testClass.Name, method.Key.Name))) cancellationTokenSource.Cancel(); if (cancellationTokenSource.IsCancellationRequested) break; } foreach (var fixture in fixtureMappings.Values.OfType<IDisposable>()) { try { fixture.Dispose(); } catch (Exception ex) { if (!messageSink.OnMessage(new ErrorMessage(ex.Unwrap()))) cancellationTokenSource.Cancel(); } } }