/// <summary> /// Initializes a new instance of the <see cref="DelegatingLongRunningTestDetectionSink"/> class, with /// long running test messages being delivered as <see cref="IDiagnosticMessage"/> instances to the /// provided diagnostic message sink. /// </summary> /// <param name="innerSink">The inner sink to delegate to.</param> /// <param name="longRunningTestTime">The minimum amount of time a test runs to be considered long running.</param> /// <param name="diagnosticMessageSink">The message sink to send messages to.</param> public DelegatingLongRunningTestDetectionSink(IExecutionSink innerSink, TimeSpan longRunningTestTime, IMessageSinkWithTypes diagnosticMessageSink) : this(innerSink, longRunningTestTime, summary => DispatchLongRunningTestsSummaryAsDiagnosticMessage(summary, diagnosticMessageSink)) { Guard.ArgumentNotNull(nameof(diagnosticMessageSink), diagnosticMessageSink); }
public void RunInteractive( string testAssemblyPath, IEnumerable <string> exampleFullNames, IExecutionSink sink) { Action <string> onExampleStarted = jsonArg => OnExampleStarted(sink, jsonArg); Action <string> onExampleCompleted = jsonArg => OnExampleCompleted(sink, jsonArg); InvokeMethod(controller, runInteractiveMethodName, testAssemblyPath, exampleFullNames, onExampleStarted, onExampleCompleted); }
/// <summary> /// Initializes a new instance of the <see cref="DelegatingXmlCreationSink"/> class. /// </summary> /// <param name="innerSink"></param> /// <param name="assemblyElement"></param> public DelegatingXmlCreationSink(IExecutionSink innerSink, XElement assemblyElement) { Guard.ArgumentNotNull(nameof(innerSink), innerSink); Guard.ArgumentNotNull(nameof(assemblyElement), assemblyElement); this.innerSink = innerSink; this.assemblyElement = assemblyElement; errorsElement = new XElement("errors"); assemblyElement.Add(errorsElement); }
static void OnExampleCompleted(IExecutionSink sink, string jsonArg) { ExecutedExample example; try { example = JsonConvert.DeserializeObject <ExecutedExample>(jsonArg); } catch (Exception ex) { throw new DotNetTestNSpecException(unknownArgumentErrorMessage .With(runInteractiveMethodName + ": " + nameof(OnExampleCompleted), jsonArg), ex); } sink.ExampleCompleted(example); }
/// <summary> /// Initializes a new instance of the <see cref="DelegatingLongRunningTestDetectionSink"/> class, with /// long running test messages being delivered as <see cref="LongRunningTestsSummary"/> to the /// provided callback. /// </summary> /// <param name="innerSink">The inner sink to delegate to.</param> /// <param name="longRunningTestTime">The minimum amount of time a test runs to be considered long running.</param> /// <param name="callback">The callback to dispatch messages to.</param> public DelegatingLongRunningTestDetectionSink(IExecutionSink innerSink, TimeSpan longRunningTestTime, Action <LongRunningTestsSummary> callback) { Guard.ArgumentNotNull(nameof(innerSink), innerSink); Guard.ArgumentValid(nameof(longRunningTestTime), "Long running test time must be at least 1 second", longRunningTestTime >= TimeSpan.FromSeconds(1)); Guard.ArgumentNotNull(nameof(callback), callback); this.innerSink = innerSink; this.longRunningTestTime = longRunningTestTime; this.callback = callback; executionSink.TestAssemblyFinishedEvent += HandleTestAssemblyFinished; executionSink.TestAssemblyStartingEvent += HandleTestAssemblyStarting; executionSink.TestCaseFinishedEvent += HandleTestCaseFinished; executionSink.TestCaseStartingEvent += HandleTestCaseStarting; }
/// <summary> /// Initializes a new instance of the <see cref="DelegatingLongRunningTestDetectionSink"/> class, with /// long running test messages being delivered as <see cref="LongRunningTestsSummary"/> to the /// provided callback. /// </summary> /// <param name="innerSink">The inner sink to delegate to.</param> /// <param name="longRunningTestTime">The minimum amount of time a test runs to be considered long running.</param> /// <param name="callback">The callback to dispatch messages to.</param> public DelegatingLongRunningTestDetectionSink(IExecutionSink innerSink, TimeSpan longRunningTestTime, Action<LongRunningTestsSummary> callback) { Guard.ArgumentNotNull(nameof(innerSink), innerSink); Guard.ArgumentValid(nameof(longRunningTestTime), "Long running test time must be at least 1 second", longRunningTestTime >= TimeSpan.FromSeconds(1)); Guard.ArgumentNotNull(nameof(callback), callback); this.innerSink = innerSink; this.longRunningTestTime = longRunningTestTime; this.callback = callback; executionSink.TestAssemblyFinishedEvent += HandleTestAssemblyFinished; executionSink.TestAssemblyStartingEvent += HandleTestAssemblyStarting; executionSink.TestCaseFinishedEvent += HandleTestCaseFinished; executionSink.TestCaseStartingEvent += HandleTestCaseStarting; }
public FailSkipSinkTests() { innerSink = Substitute.For<IExecutionSink>(); sink = new DelegatingFailSkipSink(innerSink); }
/// <summary> /// Initializes a new instance of the <see cref="DelegatingFailSkipSink"/> class. /// </summary> /// <param name="innerSink">The sink to delegate messages to.</param> public DelegatingFailSkipSink(IExecutionSink innerSink) { Guard.ArgumentNotNull(nameof(innerSink), innerSink); this.innerSink = innerSink; }
void RunTestsInAssembly(IRunContext runContext, IFrameworkHandle frameworkHandle, LoggerHelper logger, TestPlatformContext testPlatformContext, IMessageSinkWithTypes reporterMessageHandler, AssemblyRunInfo runInfo) { if (cancelled) { return; } var assemblyDisplayName = "(unknown assembly)"; try { var assembly = new XunitProjectAssembly { AssemblyFilename = runInfo.AssemblyFileName }; var assemblyFileName = runInfo.AssemblyFileName; assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName); var configuration = runInfo.Configuration; var shadowCopy = configuration.ShadowCopyOrDefault; var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; if (RunSettingsHelper.DisableAppDomain) { appDomain = AppDomainSupport.Denied; } #if WINDOWS_UAP // For AppX Apps, use the package location assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName)); #endif var diagnosticSink = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault); var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink); using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) { var testCasesMap = new Dictionary <string, TestCase>(); var testCases = new List <ITestCase>(); if (runInfo.TestCases == null || !runInfo.TestCases.Any()) { // Discover tests AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo(); DiscoverTestsInSource(controller, logger, testPlatformContext, (source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled), (source, discoverer, discoveryOptions, visitor) => { if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault()) { foreach (var testCase in visitor.TestCases) { logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')", testCase.DisplayName, testCase.UniqueID); } } assemblyDiscoveredInfo = new AssemblyDiscoveredInfo { AssemblyFileName = source, DiscoveredTestCases = GetVsTestCases(source, discoverer, visitor, logger, testPlatformContext) }; }, assemblyFileName, shadowCopy, configuration ); if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any()) { if (configuration.InternalDiagnosticMessagesOrDefault) { logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName); } return; } // Filter tests var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames)); // Apply any filtering var filter = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames); var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList(); foreach (var filteredTestCase in filteredTestCases) { var uniqueID = filteredTestCase.UniqueID; if (testCasesMap.ContainsKey(uniqueID)) { logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName); } else { testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase); testCases.Add(filteredTestCase.TestCase); } } } else { // We are in Run Specific tests scenario, the `TestCase` objects are available. // Query the `TestCase` objects to find XunitTestCase objects. foreach (var vstestCase in runInfo.TestCases) { var xunitTestCase = Deserialize(logger, controller, vstestCase); if (xunitTestCase != null) { testCasesMap.Add(xunitTestCase.UniqueID, vstestCase); testCases.Add(xunitTestCase); } } } // Execute tests var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration); if (RunSettingsHelper.DisableParallelization) { executionOptions.SetSynchronousMessageReporting(true); executionOptions.SetDisableParallelization(true); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled)) { IExecutionSink resultsSink = vsExecutionSink; if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink); } controller.RunTests(testCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); } } } catch (Exception ex) { logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex); } }
public CompletionCallbackExecutionSink(IExecutionSink innerSink, Action <ExecutionSummary> completionCallback) { _innerSink = innerSink; _completionCallback = completionCallback; }
void RunTestsInAssembly(IRunContext runContext, IFrameworkHandle frameworkHandle, LoggerHelper logger, IMessageSinkWithTypes reporterMessageHandler, AssemblyRunInfo runInfo) { if (cancelled) { return; } var assembly = new XunitProjectAssembly { AssemblyFilename = runInfo.AssemblyFileName }; var assemblyFileName = runInfo.AssemblyFileName; var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName); var shadowCopy = assembly.Configuration.ShadowCopyOrDefault; var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; if (RunSettingsHelper.DisableAppDomain) { appDomain = AppDomainSupport.Denied; } try { #if WINDOWS_UAP // For AppX Apps, use the package location assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName)); #endif var diagnosticSink = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault); using (var controller = new XunitFrontController(appDomain, assemblyFileName: assemblyFileName, configFileName: null, shadowCopy: shadowCopy, diagnosticMessageSink: MessageSinkAdapter.Wrap(diagnosticSink))) { var xunitTestCases = runInfo.TestCases.Select(tc => new { vs = tc, xunit = Deserialize(logger, controller, tc) }) .Where(tc => tc.xunit != null) .ToDictionary(tc => tc.xunit, tc => tc.vs); var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration); if (RunSettingsHelper.DisableParallelization) { executionOptions.SetSynchronousMessageReporting(true); executionOptions.SetDisableParallelization(true); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, xunitTestCases, executionOptions, () => cancelled)) { IExecutionSink resultsSink = vsExecutionSink; if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink); } controller.RunTests(xunitTestCases.Keys.ToList(), resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); } } } catch (Exception ex) { logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex); } }
public DelegatingXmlCreationSinkTests() { innerSink = Substitute.For <IExecutionSink>(); innerSink.ExecutionSummary.Returns(executionSummary); }
void RunTestsInAssembly(IRunContext runContext, IFrameworkHandle frameworkHandle, LoggerHelper logger, TestPlatformContext testPlatformContext, RunSettings runSettings, IMessageSinkWithTypes reporterMessageHandler, AssemblyRunInfo runInfo) { if (cancelled) { return; } var assemblyDisplayName = "(unknown assembly)"; try { var assembly = new XunitProjectAssembly { AssemblyFilename = runInfo.AssemblyFileName }; var assemblyFileName = runInfo.AssemblyFileName; assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName); var configuration = runInfo.Configuration; var shadowCopy = configuration.ShadowCopyOrDefault; var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior; var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault; if (runSettings.DisableAppDomain) { appDomain = AppDomainSupport.Denied; } #if WINDOWS_UAP // For AppX Apps, use the package location assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName)); #endif var diagnosticSink = DiagnosticMessageSink.ForDiagnostics(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault); var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink); using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink)) { var testCasesMap = new Dictionary <string, TestCase>(); var testCases = new List <ITestCase>(); if (runInfo.TestCases == null || !runInfo.TestCases.Any()) { // Discover tests AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo(); DiscoverTestsInSource(controller, logger, testPlatformContext, runSettings, (source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled), (source, discoverer, discoveryOptions, visitor) => { if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault()) { foreach (var testCase in visitor.TestCases) { logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')", testCase.DisplayName, testCase.UniqueID); } } assemblyDiscoveredInfo = new AssemblyDiscoveredInfo { AssemblyFileName = source, DiscoveredTestCases = GetVsTestCases(source, discoverer, visitor, logger, testPlatformContext) }; }, assemblyFileName, shadowCopy, configuration ); if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any()) { if (configuration.InternalDiagnosticMessagesOrDefault) { logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName); } return; } // Filter tests var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames)); var filter = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames); var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList(); foreach (var filteredTestCase in filteredTestCases) { var uniqueID = filteredTestCase.UniqueID; if (testCasesMap.ContainsKey(uniqueID)) { logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName); } else { testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase); testCases.Add(filteredTestCase.TestCase); } } } else { try { var serializations = runInfo.TestCases .Select(tc => tc.GetPropertyValue <string>(SerializedTestCaseProperty, null)) .ToList(); if (configuration.InternalDiagnosticMessagesOrDefault) { logger.LogWithSource(runInfo.AssemblyFileName, "Deserializing {0} test case(s):{1}{2}", serializations.Count, Environment.NewLine, string.Join(Environment.NewLine, serializations.Select(x => $" {x}"))); } var deserializedTestCasesByUniqueId = controller.BulkDeserialize(serializations); if (deserializedTestCasesByUniqueId == null) { logger.LogErrorWithSource(assemblyFileName, "Received null response from BulkDeserialize"); } else { for (int idx = 0; idx < runInfo.TestCases.Count; ++idx) { try { var kvp = deserializedTestCasesByUniqueId[idx]; var vsTestCase = runInfo.TestCases[idx]; if (kvp.Value == null) { logger.LogErrorWithSource(assemblyFileName, "Test case {0} failed to deserialize: {1}", vsTestCase.DisplayName, kvp.Key); } else { testCasesMap.Add(kvp.Key, vsTestCase); testCases.Add(kvp.Value); } } catch (Exception ex) { logger.LogErrorWithSource(assemblyFileName, "Catastrophic error deserializing item #{0}: {1}", idx, ex); } } } } catch (Exception ex) { logger.LogWarningWithSource(assemblyFileName, "Catastrophic error during deserialization: {0}", ex); } } // Execute tests var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration); if (runSettings.DisableParallelization) { executionOptions.SetSynchronousMessageReporting(true); executionOptions.SetDisableParallelization(true); } reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions)); using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled)) { IExecutionSink resultsSink = vsExecutionSink; if (longRunningSeconds > 0) { resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink); } controller.RunTests(testCases, resultsSink, executionOptions); resultsSink.Finished.WaitOne(); reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary)); } } } catch (Exception ex) { logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex); } }
/// <summary> /// Initializes a new instance of the <see cref="FailSkipSink"/> class. /// </summary> /// <param name="visitor">The visitor to pass messages onto.</param> public FailSkipSink(IExecutionSink visitor) { Guard.ArgumentNotNull(nameof(visitor), visitor); Sink = visitor; }