private static IList<TestData> Explore(IEnumerable<string> sources, ILogger frameworkLogger) { try { var testDriver = GetTestDriver(frameworkLogger); var tests = new List<TestData>(); var messageConsumer = new MessageConsumer() .Handle<TestDiscoveredMessage>(message => { if (message.Test.IsTestCase) tests.Add(message.Test); }) .Handle<AnnotationDiscoveredMessage>(message => message.Annotation.Log(frameworkLogger, true)); var loader = new ReflectionOnlyAssemblyLoader(); var assemblyInfos = sources.Select(source => LoadAssembly(source, loader)).Where(assembly => assembly != null).ToList(); var testExplorationOptions = new TestExplorationOptions(); testDriver.Describe(loader.ReflectionPolicy, assemblyInfos, testExplorationOptions, messageConsumer, NullProgressMonitor.CreateInstance()); return ResetCollectionForExposedTests(tests) ? null : tests; } catch (Exception ex) { frameworkLogger.Log(LogSeverity.Error, "Gallio failed to load tests", ex); return null; } }
/// <inheritdoc /> protected override void DescribeImpl(IReflectionPolicy reflectionPolicy, IList<ICodeElementInfo> codeElements, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor) { using (progressMonitor.BeginTask("Describing unsupported tests.", Math.Max(codeElements.Count, 1))) { PublishTestModelFromCodeElements(codeElements, messageSink, progressMonitor); } }
public void SetUp() { logger = MockRepository.GenerateStub<ILogger>(); driver = new IronRubyTestDriver(logger); var testIsolationProvider = (ITestIsolationProvider)RuntimeAccessor.ServiceLocator.ResolveByComponentId("Gallio.LocalTestIsolationProvider"); testIsolationContext = testIsolationProvider.CreateContext(new TestIsolationOptions(), logger); testPackage = new TestPackage(); testExplorationOptions = new TestExplorationOptions(); testExecutionOptions = new TestExecutionOptions(); messageSink = MockRepository.GenerateStub<IMessageSink>(); progressMonitor = NullProgressMonitor.CreateInstance(); }
/// <summary> /// Initializes the event arguments. /// </summary> /// <param name="testPackage">The test package configuration.</param> /// <param name="testExplorationOptions">The test exploration options.</param> /// <param name="reportLockBox">The report lock-box which may be used to access the report asynchronously during execution.</param> /// <exception cref="ArgumentNullException">Thrown if <paramref name="testPackage"/> /// or <paramref name="testExplorationOptions"/> is null.</exception> public ExploreStartedEventArgs(TestPackage testPackage, TestExplorationOptions testExplorationOptions, LockBox<Report> reportLockBox) { if (testPackage == null) throw new ArgumentNullException("testPackage"); if (testExplorationOptions == null) throw new ArgumentNullException("testExplorationOptions"); this.testPackage = testPackage; this.testExplorationOptions = testExplorationOptions; this.reportLockBox = reportLockBox; }
/// <inheritdoc /> public void Describe(IReflectionPolicy reflectionPolicy, IList<ICodeElementInfo> codeElements, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor) { if (reflectionPolicy == null) throw new ArgumentNullException("reflectionPolicy"); if (codeElements == null || codeElements.Contains(null)) throw new ArgumentNullException("codeElements"); if (testExplorationOptions == null) throw new ArgumentNullException("testExplorationOptions"); if (messageSink == null) throw new ArgumentNullException("messageSink"); if (progressMonitor == null) throw new ArgumentNullException("progressMonitor"); DescribeImpl(reflectionPolicy, codeElements, testExplorationOptions, messageSink, progressMonitor); }
/// <inheritdoc /> public void Explore(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor) { if (testIsolationContext == null) throw new ArgumentNullException("testIsolationContext"); if (testPackage == null) throw new ArgumentNullException("testPackage"); if (testExplorationOptions == null) throw new ArgumentNullException("testExplorationOptions"); if (messageSink == null) throw new ArgumentNullException("messageSink"); if (progressMonitor == null) throw new ArgumentNullException("progressMonitor"); ExploreImpl(testIsolationContext, testPackage, testExplorationOptions, messageSink, progressMonitor); }
public override ICollection Load(string location, ProjectData projectData, IWarningHandler warningHandler) { // Skip loading if the extension is not fully initalized unless we are not // running in Visual Studio (because we are running in MSTest instead). if (!TipShellExtension.IsInitialized && ShellEnvironment.IsRunningInVisualStudio) return EmptyArray<TestElement>.Instance; // Explore the tests. ITestFrameworkManager testFrameworkManager = RuntimeAccessor.ServiceLocator.Resolve<ITestFrameworkManager>(); WarningLogger logger = new WarningLogger(warningHandler); ReflectionOnlyAssemblyLoader loader = new ReflectionOnlyAssemblyLoader(); loader.AddHintDirectory(Path.GetDirectoryName(location)); IAssemblyInfo assembly = loader.ReflectionPolicy.LoadAssemblyFrom(location); var testFrameworkSelector = new TestFrameworkSelector() { Filter = testFrameworkHandle => testFrameworkHandle.Id != "MSTestAdapter.TestFramework", FallbackMode = TestFrameworkFallbackMode.Approximate }; ITestDriver driver = testFrameworkManager.GetTestDriver(testFrameworkSelector, logger); TestExplorationOptions testExplorationOptions = new TestExplorationOptions(); ArrayList tests = new ArrayList(); MessageConsumer messageConsumer = new MessageConsumer() .Handle<TestDiscoveredMessage>(message => { if (message.Test.IsTestCase) tests.Add(GallioTestElementFactory.CreateTestElement(message.Test, location, projectData)); }) .Handle<AnnotationDiscoveredMessage>(message => { message.Annotation.Log(logger, true); }); driver.Describe(loader.ReflectionPolicy, new ICodeElementInfo[] { assembly }, testExplorationOptions, messageConsumer, NullProgressMonitor.CreateInstance()); return tests; }
private void ExploreOrRun(Type driverType, object[] driverArguments, string assemblyPath, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor) { Assembly assembly = LoadAssembly(assemblyPath); using (var queuedMessageSink = new QueuedMessageSink(messageSink)) { var testDriver = (DotNetTestDriver)Activator.CreateInstance(driverType, driverArguments); if (testExecutionOptions == null) { testDriver.ExploreAssembly(assembly, testExplorationOptions, queuedMessageSink, progressMonitor); } else { testDriver.RunAssembly(assembly, testExplorationOptions, testExecutionOptions, queuedMessageSink, progressMonitor); } } }
protected override void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor) { using (progressMonitor.BeginTask("Running tests.", 1)) { if (testIsolationContext.RequiresSingleThreadedExecution && !testExecutionOptions.SingleThreaded) { testExecutionOptions = testExecutionOptions.Copy(); testExecutionOptions.SingleThreaded = true; } ForEachDriver(testFrameworkManager.SelectTestFrameworksForFilesImpl( testFrameworkHandles, testFrameworkFallbackMode, testFrameworkOptions, testPackage.Files), (driver, items, driverCount) => { TestPackage testPackageForDriver = CreateTestPackageWithFiles(testPackage, items); driver.Run(testIsolationContext, testPackageForDriver, testExplorationOptions, testExecutionOptions, messageSink, progressMonitor.CreateSubProgressMonitor(1.0 / driverCount)); return(false); }); } }
/// <summary> /// Runs tests in an assembly. /// </summary> /// <remarks> /// <para> /// The default implementation does nothing. Subclasses may override to enable tests /// to be run. This is required for actually running tests. /// </para> /// <para> /// This method executes within the test host which is most likely a different AppDomain /// or Process from the test runner itself. /// </para> /// </remarks> /// <param name="assembly">The test assembly, not null.</param> /// <param name="testExplorationOptions">The test exploration options, not null.</param> /// <param name="testExecutionOptions">The test execution options, not null.</param> /// <param name="messageSink">The message sink, not null.</param> /// <param name="progressMonitor">The progress monitor, not null.</param> protected virtual void RunAssembly(Assembly assembly, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor) { }
/// <summary> /// Creates a copy of the options. /// </summary> /// <returns>The copy.</returns> public TestExplorationOptions Copy() { var copy = new TestExplorationOptions(); copy.properties.AddAll(properties); return copy; }
private static List<TestData> FindTests(TestPackage testPackage, ITestRunner runner) { var options = new TestExplorationOptions(); testPackage.AddExcludedTestFrameworkId(MsTestFrameworkHandleId); testPackage.AddExcludedTestFrameworkId(NUnitTestFrameworkHandleId); var result = runner.Explore(testPackage, options, new ObservableProgressMonitor()); var tests = result.TestModel.AllTests.Where(t => t.IsTestCase).ToList(); return tests; }
/// <inheritdoc /> public void Explore(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor) { if (testIsolationContext == null) { throw new ArgumentNullException("testIsolationContext"); } if (testPackage == null) { throw new ArgumentNullException("testPackage"); } if (testExplorationOptions == null) { throw new ArgumentNullException("testExplorationOptions"); } if (messageSink == null) { throw new ArgumentNullException("messageSink"); } if (progressMonitor == null) { throw new ArgumentNullException("progressMonitor"); } ExploreImpl(testIsolationContext, testPackage, testExplorationOptions, messageSink, progressMonitor); }
/// <summary> /// Runs tests from a test package. /// </summary> /// <remarks> /// <para> /// The default implementation does nothing. Subclasses may override to enable tests /// to be run. This is required for actually running tests. /// </para> /// </remarks> /// <param name="testIsolationContext">The test isolation context, not null.</param> /// <param name="testPackage">The test package, not null.</param> /// <param name="testExplorationOptions">The test exploration options, not null.</param> /// <param name="testExecutionOptions">The test execution options, not null.</param> /// <param name="messageSink">The message sink to receive test exploration and execution messages, not null.</param> /// <param name="progressMonitor">The progress monitor, not null.</param> /// <returns>The test report.</returns> protected virtual void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor) { }
/// <inheritdoc /> protected override void ExploreImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor) { using (progressMonitor.BeginTask("Exploring unsupported tests.", Math.Max(testPackage.Files.Count, 1))) { PublishTestModelFromFiles(testPackage.Files, messageSink, progressMonitor); } }
private static void ExploreOrRun(TestPackage testPackage, ScriptRuntimeSetup scriptRuntimeSetup, string testDriverScriptPath, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor, ILogger logger) { using (BufferedLogWriter outputWriter = new BufferedLogWriter(logger, LogSeverity.Info, Encoding.Default), errorWriter = new BufferedLogWriter(logger, LogSeverity.Error, Encoding.Default)) { using (var queuedMessageSink = new QueuedMessageSink(messageSink)) { using (new ConsoleRedirection(outputWriter, errorWriter)) { var scriptRuntime = new ScriptRuntime(scriptRuntimeSetup); scriptRuntime.IO.SetInput(Stream.Null, TextReader.Null, Encoding.Default); scriptRuntime.IO.SetOutput(new TextWriterStream(outputWriter), outputWriter); scriptRuntime.IO.SetErrorOutput(new TextWriterStream(errorWriter), errorWriter); try { var scriptParameters = new Dictionary<string, object>(); scriptParameters.Add("Verb", testExecutionOptions != null ? "Run" : "Explore"); scriptParameters.Add("TestPackage", testPackage); scriptParameters.Add("TestExplorationOptions", testExplorationOptions); scriptParameters.Add("TestExecutionOptions", testExecutionOptions); scriptParameters.Add("MessageSink", queuedMessageSink); scriptParameters.Add("ProgressMonitor", progressMonitor); scriptParameters.Add("Logger", logger); scriptRuntime.Globals.SetVariable(ScriptParametersVariableName, scriptParameters); RunScript(scriptRuntime, testDriverScriptPath); } finally { scriptRuntime.Shutdown(); } } } } }
private void ExploreOrRun(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor, string taskName) { using (progressMonitor.BeginTask(taskName, 1)) { if (progressMonitor.IsCanceled) return; FileInfo testDriverScriptFile = GetTestDriverScriptFile(testPackage); if (testDriverScriptFile == null) return; HostSetup hostSetup = CreateHostSetup(testPackage); ScriptRuntimeSetup scriptRuntimeSetup = CreateScriptRuntimeSetup(testPackage); string testDriverScriptPath = testDriverScriptFile.FullName; var remoteMessageSink = new RemoteMessageSink(messageSink); var remoteLogger = new RemoteLogger(logger); using (var remoteProgressMonitor = new RemoteProgressMonitor(progressMonitor.CreateSubProgressMonitor(1))) { testIsolationContext.RunIsolatedTask<ExploreOrRunTask>(hostSetup, (statusMessage) => progressMonitor.SetStatus(statusMessage), new object[] { testPackage, scriptRuntimeSetup, testDriverScriptPath, testExplorationOptions, testExecutionOptions, remoteMessageSink, remoteProgressMonitor, remoteLogger }); } } }
private void ExploreOrRun(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor, string taskName) { using (progressMonitor.BeginTask(taskName, Math.Max(testPackage.Files.Count, 1))) { foreach (FileInfo file in testPackage.Files) { if (progressMonitor.IsCanceled) { return; } RemoteMessageSink remoteMessageSink = new RemoteMessageSink(messageSink); ExploreOrRunAssembly(testIsolationContext, testPackage, testExplorationOptions, testExecutionOptions, remoteMessageSink, progressMonitor.CreateSubProgressMonitor(1), taskName, file); } } }
private void ExploreOrRunAssembly(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions, RemoteMessageSink remoteMessageSink, IProgressMonitor progressMonitor, string taskName, FileInfo file) { using (progressMonitor.BeginTask(taskName, 100)) { if (progressMonitor.IsCanceled) { return; } string assemblyPath = file.FullName; progressMonitor.SetStatus("Getting test assembly metadata."); AssemblyMetadata assemblyMetadata = AssemblyUtils.GetAssemblyMetadata(assemblyPath, AssemblyMetadataFields.RuntimeVersion); progressMonitor.Worked(2); if (progressMonitor.IsCanceled) { return; } if (assemblyMetadata != null) { Type driverType = GetType(); object[] driverArguments = GetRemoteTestDriverArguments(); HostSetup hostSetup = CreateHostSetup(testPackage, assemblyPath, assemblyMetadata); using (var remoteProgressMonitor = new RemoteProgressMonitor( progressMonitor.CreateSubProgressMonitor(97))) { testIsolationContext.RunIsolatedTask <ExploreOrRunTask>(hostSetup, (statusMessage) => progressMonitor.SetStatus(statusMessage), new object[] { driverType, driverArguments, assemblyPath, testExplorationOptions, testExecutionOptions, remoteMessageSink, remoteProgressMonitor }); } // Record one final work unit after the isolated task has been fully cleaned up. progressMonitor.SetStatus(""); progressMonitor.Worked(1); } } }
/// <summary> /// Describes tests via reflection over code elements. /// </summary> /// <remarks> /// <para> /// The default implementation does nothing. Subclasses may override to enable tests /// to be described via reflection only. This is required for ReSharper integration. /// </para> /// </remarks> /// <param name="reflectionPolicy">The reflection policy, not null.</param> /// <param name="codeElements">The enumeration of code elements, usually <see cref="ITypeInfo" /> /// and <see cref="IAssemblyInfo" /> objects, that might contain tests to be described, not null.</param> /// <param name="testExplorationOptions">The test exploration options, not null.</param> /// <param name="messageSink">The message sink to receive test exploration messages, not null.</param> /// <param name="progressMonitor">The progress monitor, not null.</param> protected virtual void DescribeImpl(IReflectionPolicy reflectionPolicy, IList<ICodeElementInfo> codeElements, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor) { }
/// <inheritdoc /> protected override void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor) { double workItems = Math.Max(testPackage.Files.Count, 1); using (progressMonitor.BeginTask("Running unsupported tests.", workItems * 2)) { TestModel testModel = PublishTestModelFromFiles(testPackage.Files, messageSink, progressMonitor); if (testModel != null) RunTestModel(testModel, messageSink, progressMonitor, workItems); } }
protected override void ExploreImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor) { using (progressMonitor.BeginTask("Exploring tests.", 1)) { ForEachDriver(testFrameworkManager.SelectTestFrameworksForFilesImpl( testFrameworkHandles, testFrameworkFallbackMode, testFrameworkOptions, testPackage.Files), (driver, items, driverCount) => { TestPackage testPackageForDriver = CreateTestPackageWithFiles(testPackage, items); driver.Explore(testIsolationContext, testPackageForDriver, testExplorationOptions, messageSink, progressMonitor.CreateSubProgressMonitor(1.0 / driverCount)); return(false); }); } }
/// <inheritdoc /> sealed protected override void ExploreImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor) { ExploreOrRun(testIsolationContext, testPackage, testExplorationOptions, null, messageSink, progressMonitor, "Exploring tests."); }
/// <summary> /// Describes tests via reflection over code elements. /// </summary> /// <remarks> /// <para> /// The default implementation does nothing. Subclasses may override to enable tests /// to be described via reflection only. This is required for ReSharper integration. /// </para> /// </remarks> /// <param name="reflectionPolicy">The reflection policy, not null.</param> /// <param name="codeElements">The enumeration of code elements, usually <see cref="ITypeInfo" /> /// and <see cref="IAssemblyInfo" /> objects, that might contain tests to be described, not null.</param> /// <param name="testExplorationOptions">The test exploration options, not null.</param> /// <param name="messageSink">The message sink to receive test exploration messages, not null.</param> /// <param name="progressMonitor">The progress monitor, not null.</param> protected virtual void DescribeImpl(IReflectionPolicy reflectionPolicy, IList <ICodeElementInfo> codeElements, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor) { }
/// <inheritdoc /> public void Describe(IReflectionPolicy reflectionPolicy, IList <ICodeElementInfo> codeElements, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor) { if (reflectionPolicy == null) { throw new ArgumentNullException("reflectionPolicy"); } if (codeElements == null || codeElements.Contains(null)) { throw new ArgumentNullException("codeElements"); } if (testExplorationOptions == null) { throw new ArgumentNullException("testExplorationOptions"); } if (messageSink == null) { throw new ArgumentNullException("messageSink"); } if (progressMonitor == null) { throw new ArgumentNullException("progressMonitor"); } DescribeImpl(reflectionPolicy, codeElements, testExplorationOptions, messageSink, progressMonitor); }
public IEnumerable<AutoTest.TestRunners.Shared.Results.TestResult> Run(RunSettings settings) { if (!_isInitialized) return new AutoTest.TestRunners.Shared.Results.TestResult[] { getNotInitializedResult(settings) }; var tests = settings.Assembly.Tests.ToList(); var members = settings.Assembly.Members.ToList(); var namespaces = settings.Assembly.Namespaces.ToList(); var runAll = namespaces.Count == 0 && members.Count == 0 && tests.Count == 0; var steps = new List<TestStepData>(); var testResults = new List<AutoTest.TestRunners.Shared.Results.TestResult>(); // Get a test isolation context. Here we want to run tests in the same AppDomain. var testIsolationProvider = (ITestIsolationProvider)RuntimeAccessor.ServiceLocator.ResolveByComponentId("Gallio.LocalTestIsolationProvider"); var testIsolationOptions = new TestIsolationOptions(); ITestIsolationContext testIsolationContext = testIsolationProvider.CreateContext(testIsolationOptions, _logger); var testPackage = new TestPackage(); testPackage.AddFile(new FileInfo(settings.Assembly.Assembly)); testPackage.TestFrameworkFallbackMode = TestFrameworkFallbackMode.Strict; // Create some test exploration options. Nothing interesting for you here, probably. var testExplorationOptions = new TestExplorationOptions(); var messageSink = new MessageConsumer() .Handle<TestStepStartedMessage>((message) => { steps.Add(message.Step); }) .Handle<TestStepFinishedMessage>(message => { var test = steps.FirstOrDefault(x => x.Id.Equals(message.StepId) && x.IsTestCase); if (test == null) return; var fixture = string.Format("{0}.{1}", test.CodeReference.NamespaceName, steps.First(x => x.Id.Equals(test.ParentId)).Name); testResults.Add(new AutoTest.TestRunners.Shared.Results.TestResult( "MbUnit", settings.Assembly.Assembly, fixture, message.Result.Duration.TotalMilliseconds, string.Format("{0}.{1}", fixture, test.Name), convertState(message.Result.Outcome.Status), message.Result.Outcome.DisplayName)); }); // Provide a progress monitor. var logProgressMonitorProvider = new LogProgressMonitorProvider(_logger); var options = new TestExecutionOptions(); options.FilterSet = new Gallio.Model.Filters.FilterSet<ITestDescriptor>(new OrFilter<ITestDescriptor>(getTestFilter(namespaces, members, tests))); // Run the tests. logProgressMonitorProvider.Run((progressMonitor) => { _testDriver.Run(testIsolationContext, testPackage, testExplorationOptions, options, messageSink, progressMonitor); }); return testResults; }
/// <inheritdoc /> public Report Run(TestPackage testPackage, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions, IProgressMonitor progressMonitor) { if (testPackage == null) throw new ArgumentNullException("testPackageConfig"); if (testExplorationOptions == null) throw new ArgumentNullException("testExplorationOptions"); if (testExecutionOptions == null) throw new ArgumentNullException("testExecutionOptions"); if (progressMonitor == null) throw new ArgumentNullException("progressMonitor"); ThrowIfDisposed(); if (state != State.Initialized) throw new InvalidOperationException("The test runner must be initialized before this operation is performed."); testPackage = testPackage.Copy(); testExplorationOptions = testExplorationOptions.Copy(); testExecutionOptions = testExecutionOptions.Copy(); GenericCollectionUtils.ForEach(testRunnerOptions.Properties, x => testPackage.AddProperty(x.Key, x.Value)); using (progressMonitor.BeginTask("Running the tests.", 10)) { Stopwatch stopwatch = Stopwatch.StartNew(); Report report = new Report() { TestPackage = new TestPackageData(testPackage), TestModel = new TestModelData(), TestPackageRun = new TestPackageRun() { StartTime = DateTime.Now } }; var reportLockBox = new LockBox<Report>(report); eventDispatcher.NotifyRunStarted(new RunStartedEventArgs(testPackage, testExplorationOptions, testExecutionOptions, reportLockBox)); bool success; using (Listener listener = new Listener(eventDispatcher, tappedLogger, reportLockBox)) { try { ITestDriver testDriver = testFrameworkManager.GetTestDriver( testPackage.CreateTestFrameworkSelector(), tappedLogger); using (testIsolationContext.BeginBatch(progressMonitor.SetStatus)) { testDriver.Run(testIsolationContext, testPackage, testExplorationOptions, testExecutionOptions, listener, progressMonitor.CreateSubProgressMonitor(10)); } success = true; } catch (Exception ex) { success = false; tappedLogger.Log(LogSeverity.Error, "A fatal exception occurred while running tests. Possible causes include invalid test runner parameters and stack overflows.", ex); report.TestModel.Annotations.Add(new AnnotationData(AnnotationType.Error, CodeLocation.Unknown, CodeReference.Unknown, "A fatal exception occurred while running tests. See log for details.", null)); } finally { report.TestPackageRun.EndTime = DateTime.Now; report.TestPackageRun.Statistics.Duration = stopwatch.Elapsed.TotalSeconds; } } eventDispatcher.NotifyRunFinished(new RunFinishedEventArgs(success, report)); return report; } }
/// <inheritdoc /> sealed protected override void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor) { ExploreOrRun(testIsolationContext, testPackage, testExplorationOptions, testExecutionOptions, messageSink, progressMonitor, "Running tests."); }
private void RunAllTests(IRunContext runContext) { ITestRunnerManager runnerManager = RuntimeAccessor.ServiceLocator.Resolve<ITestRunnerManager>(); var runner = runnerManager.CreateTestRunner(StandardTestRunnerFactoryNames.IsolatedAppDomain); runner.RegisterExtension(new RunContextExtension(runContext)); ILogger logger = new RunContextLogger(runContext); TestRunnerOptions testRunnerOptions = new TestRunnerOptions(); try { RunWithProgressMonitor(delegate(IProgressMonitor progressMonitor) { runner.Initialize(testRunnerOptions, logger, progressMonitor); }); if (isCanceled) return; TestPackage testPackage = new TestPackage(); testPackage.AddExcludedTestFrameworkId("MSTestAdapter.TestFramework"); foreach (ITestElement testElement in runContext.RunConfig.TestElements) { GallioTestElement gallioTestElement = testElement as GallioTestElement; if (gallioTestElement != null) { testPackage.AddFile(new FileInfo(gallioTestElement.AssemblyPath)); } } TestExplorationOptions testExplorationOptions = new TestExplorationOptions(); TestExecutionOptions testExecutionOptions = new TestExecutionOptions(); List<Filter<string>> idFilters = new List<Filter<string>>(); foreach (ITestElement includedTestElement in runContext.RunConfig.TestElements) { GallioTestElement gallioTestElement = includedTestElement as GallioTestElement; if (gallioTestElement != null) idFilters.Add(new EqualityFilter<string>(gallioTestElement.GallioTestId)); } testExecutionOptions.FilterSet = new FilterSet<ITestDescriptor>(new IdFilter<ITestDescriptor>(new OrFilter<string>(idFilters))); RunWithProgressMonitor(delegate(IProgressMonitor progressMonitor) { runner.Run(testPackage, testExplorationOptions, testExecutionOptions, progressMonitor); }); } finally { runner.Dispose(NullProgressMonitor.CreateInstance()); } }
TestModel PopulateTestTree(Assembly assembly) { TestModel testModel = new TestModel(); var testFrameworkManager = RuntimeAccessor.ServiceLocator.Resolve<ITestFrameworkManager>(); var logger = new MarkupStreamLogger(TestLog.Default); var testFrameworkSelector = new TestFrameworkSelector() { Filter = testFrameworkHandle => testFrameworkHandle.Id == TestFrameworkHandle.Id, FallbackMode = TestFrameworkFallbackMode.Strict }; ITestDriver testDriver = testFrameworkManager.GetTestDriver(testFrameworkSelector, logger); var testIsolationProvider = (ITestIsolationProvider)RuntimeAccessor.ServiceLocator.ResolveByComponentId("Gallio.LocalTestIsolationProvider"); var testIsolationOptions = new TestIsolationOptions(); using (ITestIsolationContext testIsolationContext = testIsolationProvider.CreateContext(testIsolationOptions, logger)) { var testPackage = new TestPackage(); testPackage.AddFile(new FileInfo(AssemblyUtils.GetFriendlyAssemblyCodeBase(assembly))); var testExplorationOptions = new TestExplorationOptions(); var messageSink = TestModelSerializer.CreateMessageSinkToPopulateTestModel(testModel); new LogProgressMonitorProvider(logger).Run(progressMonitor => { testDriver.Explore(testIsolationContext, testPackage, testExplorationOptions, messageSink, progressMonitor); }); } return testModel; }
/// <inheritdoc /> protected override sealed void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor) { ExploreOrRun(testIsolationContext, testPackage, testExplorationOptions, testExecutionOptions, messageSink, progressMonitor, "Running tests."); }
protected override void DescribeImpl(IReflectionPolicy reflectionPolicy, IList <ICodeElementInfo> codeElements, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, Gallio.Runtime.ProgressMonitoring.IProgressMonitor progressMonitor) { using (progressMonitor.BeginTask("Describing tests.", 1)) { ForEachDriver(testFrameworkManager.SelectTestFrameworksForCodeElementsImpl( testFrameworkHandles, testFrameworkFallbackMode, testFrameworkOptions, codeElements), (driver, items, driverCount) => { driver.Describe(reflectionPolicy, items, testExplorationOptions, messageSink, progressMonitor.CreateSubProgressMonitor(1.0 / driverCount)); return(false); }); } }
private List<Test> getTests(string assembly) { var testIsolationProvider = (ITestIsolationProvider)RuntimeAccessor.ServiceLocator.ResolveByComponentId("Gallio.LocalTestIsolationProvider"); var testIsolationOptions = new TestIsolationOptions(); ITestIsolationContext testIsolationContext = testIsolationProvider.CreateContext(testIsolationOptions, _logger); // Create a test package. // You can set a whole bunch of options here. var testPackage = new TestPackage(); testPackage.AddFile(new FileInfo(assembly)); testPackage.TestFrameworkFallbackMode = TestFrameworkFallbackMode.Strict; var testExplorationOptions = new TestExplorationOptions(); // This query you can answer by exploring tests and looking at their metadata for "TestsOn". // You can explore tests using the Explore method of TestDriver. It sends a bunch of // messages to an IMessageSink just like Run. Just scan these messages for tests that // match the pattern you like. // Alternately, you can build a TestModel once up front and traverse the resulting test tree // to find what you need. The Explore method of testDriver is just like Run, except that // it does not run the tests. // TestModelSerializer is a little helper we can use to build up a TestModel from test messages on the fly. // The TestModel is just a tree of test metadata. Pretty straightforward. TestModel testModel = new TestModel(); IMessageSink messageSink = TestModelSerializer.CreateMessageSinkToPopulateTestModel(testModel); var logProgressMonitorProvider = new LogProgressMonitorProvider(_logger); logProgressMonitorProvider.Run((progressMonitor) => { _testDriver.Explore(testIsolationContext, testPackage, testExplorationOptions, messageSink, progressMonitor); }); return testModel.AllTests.Where(x => x.IsTestCase).ToList(); }
private void ExploreOrRun(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor, string taskName) { using (progressMonitor.BeginTask(taskName, Math.Max(testPackage.Files.Count, 1))) { foreach (FileInfo file in testPackage.Files) { if (progressMonitor.IsCanceled) return; RemoteMessageSink remoteMessageSink = new RemoteMessageSink(messageSink); ExploreOrRunAssembly(testIsolationContext, testPackage, testExplorationOptions, testExecutionOptions, remoteMessageSink, progressMonitor.CreateSubProgressMonitor(1), taskName, file); } } }
private void Describe(IReflectionPolicy reflectionPolicy, IList<ICodeElementInfo> codeElements, ConsumerAdapter consumerAdapter) { var testDriver = CreateTestDriver(); var testExplorationOptions = new TestExplorationOptions(); testDriver.Describe(reflectionPolicy, codeElements, testExplorationOptions, consumerAdapter, NullProgressMonitor.CreateInstance()); }
private void ExploreOrRunAssembly(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions, RemoteMessageSink remoteMessageSink, IProgressMonitor progressMonitor, string taskName, FileInfo file) { using (progressMonitor.BeginTask(taskName, 100)) { if (progressMonitor.IsCanceled) return; string assemblyPath = file.FullName; progressMonitor.SetStatus("Getting test assembly metadata."); AssemblyMetadata assemblyMetadata = AssemblyUtils.GetAssemblyMetadata(assemblyPath, AssemblyMetadataFields.RuntimeVersion); progressMonitor.Worked(2); if (progressMonitor.IsCanceled) return; if (assemblyMetadata != null) { Type driverType = GetType(); object[] driverArguments = GetRemoteTestDriverArguments(); HostSetup hostSetup = CreateHostSetup(testPackage, assemblyPath, assemblyMetadata); using (var remoteProgressMonitor = new RemoteProgressMonitor( progressMonitor.CreateSubProgressMonitor(97))) { testIsolationContext.RunIsolatedTask<ExploreOrRunTask>(hostSetup, (statusMessage) => progressMonitor.SetStatus(statusMessage), new object[] { driverType, driverArguments, assemblyPath, testExplorationOptions, testExecutionOptions, remoteMessageSink, remoteProgressMonitor }); } // Record one final work unit after the isolated task has been fully cleaned up. progressMonitor.SetStatus(""); progressMonitor.Worked(1); } } }
private void ExploreOrRun(Type driverType, object[] driverArguments, string assemblyPath, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor) { Assembly assembly = LoadAssembly(assemblyPath); using (var queuedMessageSink = new QueuedMessageSink(messageSink)) { var testDriver = (DotNetTestDriver) Activator.CreateInstance(driverType, driverArguments); if (testExecutionOptions == null) testDriver.ExploreAssembly(assembly, testExplorationOptions, queuedMessageSink, progressMonitor); else testDriver.RunAssembly(assembly, testExplorationOptions, testExecutionOptions, queuedMessageSink, progressMonitor); } }
private FacadeTaskResult RunTests() { var logger = new FacadeLoggerWrapper(facadeLogger); var runner = TestRunnerUtils.CreateTestRunnerByName(StandardTestRunnerFactoryNames.IsolatedAppDomain); // Set parameters. var testPackage = new TestPackage(); foreach (var assemblyLocation in assemblyLocations) testPackage.AddFile(new FileInfo(assemblyLocation)); testPackage.ShadowCopy = facadeTaskExecutorConfiguration.ShadowCopy; if (facadeTaskExecutorConfiguration.AssemblyFolder != null) { testPackage.ApplicationBaseDirectory = new DirectoryInfo(facadeTaskExecutorConfiguration.AssemblyFolder); testPackage.WorkingDirectory = new DirectoryInfo(facadeTaskExecutorConfiguration.AssemblyFolder); } var testRunnerOptions = new TestRunnerOptions(); var testExplorationOptions = new TestExplorationOptions(); var filters = GenericCollectionUtils.ConvertAllToArray<string, Filter<string>>(explicitTestIds, testId => new EqualityFilter<string>(testId)); var filterSet = new FilterSet<ITestDescriptor>(new IdFilter<ITestDescriptor>(new OrFilter<string>(filters))); var testExecutionOptions = new TestExecutionOptions { FilterSet = filterSet }; // Install the listeners. runner.Events.TestStepStarted += TestStepStarted; runner.Events.TestStepFinished += TestStepFinished; runner.Events.TestStepLifecyclePhaseChanged += TestStepLifecyclePhaseChanged; // Run the tests. try { try { runner.Initialize(testRunnerOptions, logger, CreateProgressMonitor()); Report report = runner.Run(testPackage, testExplorationOptions, testExecutionOptions, CreateProgressMonitor()); if (sessionId != null) SessionCache.SaveSerializedReport(sessionId, report); return FacadeTaskResult.Success; } catch (Exception ex) { if (sessionId != null) SessionCache.ClearSerializedReport(sessionId); logger.Log(LogSeverity.Error, "A fatal exception occurred during test execution.", ex); return FacadeTaskResult.Exception; } finally { SubmitFailureForRemainingPendingTasks(); } } finally { runner.Dispose(CreateProgressMonitor()); } }