private static IList<TestData> Explore(IEnumerable<string> sources, ILogger frameworkLogger)
{
try
{
var testDriver = GetTestDriver(frameworkLogger);
var tests = new List<TestData>();
var messageConsumer = new MessageConsumer()
.Handle<TestDiscoveredMessage>(message =>
{
if (message.Test.IsTestCase)
tests.Add(message.Test);
})
.Handle<AnnotationDiscoveredMessage>(message => message.Annotation.Log(frameworkLogger, true));
var loader = new ReflectionOnlyAssemblyLoader();
var assemblyInfos = sources.Select(source => LoadAssembly(source, loader)).Where(assembly => assembly != null).ToList();
var testExplorationOptions = new TestExplorationOptions();
testDriver.Describe(loader.ReflectionPolicy, assemblyInfos, testExplorationOptions, messageConsumer, NullProgressMonitor.CreateInstance());
return ResetCollectionForExposedTests(tests) ? null : tests;
}
catch (Exception ex)
{
frameworkLogger.Log(LogSeverity.Error, "Gallio failed to load tests", ex);
return null;
}
}
/// <inheritdoc />
protected override void DescribeImpl(IReflectionPolicy reflectionPolicy, IList<ICodeElementInfo> codeElements,
TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
{
using (progressMonitor.BeginTask("Describing unsupported tests.", Math.Max(codeElements.Count, 1)))
{
PublishTestModelFromCodeElements(codeElements, messageSink, progressMonitor);
}
}
public void SetUp()
{
logger = MockRepository.GenerateStub<ILogger>();
driver = new IronRubyTestDriver(logger);
var testIsolationProvider = (ITestIsolationProvider)RuntimeAccessor.ServiceLocator.ResolveByComponentId("Gallio.LocalTestIsolationProvider");
testIsolationContext = testIsolationProvider.CreateContext(new TestIsolationOptions(), logger);
testPackage = new TestPackage();
testExplorationOptions = new TestExplorationOptions();
testExecutionOptions = new TestExecutionOptions();
messageSink = MockRepository.GenerateStub<IMessageSink>();
progressMonitor = NullProgressMonitor.CreateInstance();
}
/// <summary>
/// Initializes the event arguments.
/// </summary>
/// <param name="testPackage">The test package configuration.</param>
/// <param name="testExplorationOptions">The test exploration options.</param>
/// <param name="reportLockBox">The report lock-box which may be used to access the report asynchronously during execution.</param>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="testPackage"/>
/// or <paramref name="testExplorationOptions"/> is null.</exception>
public ExploreStartedEventArgs(TestPackage testPackage, TestExplorationOptions testExplorationOptions,
LockBox<Report> reportLockBox)
{
if (testPackage == null)
throw new ArgumentNullException("testPackage");
if (testExplorationOptions == null)
throw new ArgumentNullException("testExplorationOptions");
this.testPackage = testPackage;
this.testExplorationOptions = testExplorationOptions;
this.reportLockBox = reportLockBox;
}
/// <inheritdoc />
public void Describe(IReflectionPolicy reflectionPolicy, IList<ICodeElementInfo> codeElements, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
{
if (reflectionPolicy == null)
throw new ArgumentNullException("reflectionPolicy");
if (codeElements == null || codeElements.Contains(null))
throw new ArgumentNullException("codeElements");
if (testExplorationOptions == null)
throw new ArgumentNullException("testExplorationOptions");
if (messageSink == null)
throw new ArgumentNullException("messageSink");
if (progressMonitor == null)
throw new ArgumentNullException("progressMonitor");
DescribeImpl(reflectionPolicy, codeElements, testExplorationOptions, messageSink, progressMonitor);
}
/// <inheritdoc />
public void Explore(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
IMessageSink messageSink, IProgressMonitor progressMonitor)
{
if (testIsolationContext == null)
throw new ArgumentNullException("testIsolationContext");
if (testPackage == null)
throw new ArgumentNullException("testPackage");
if (testExplorationOptions == null)
throw new ArgumentNullException("testExplorationOptions");
if (messageSink == null)
throw new ArgumentNullException("messageSink");
if (progressMonitor == null)
throw new ArgumentNullException("progressMonitor");
ExploreImpl(testIsolationContext, testPackage, testExplorationOptions, messageSink, progressMonitor);
}
public override ICollection Load(string location, ProjectData projectData, IWarningHandler warningHandler)
{
// Skip loading if the extension is not fully initalized unless we are not
// running in Visual Studio (because we are running in MSTest instead).
if (!TipShellExtension.IsInitialized && ShellEnvironment.IsRunningInVisualStudio)
return EmptyArray<TestElement>.Instance;
// Explore the tests.
ITestFrameworkManager testFrameworkManager = RuntimeAccessor.ServiceLocator.Resolve<ITestFrameworkManager>();
WarningLogger logger = new WarningLogger(warningHandler);
ReflectionOnlyAssemblyLoader loader = new ReflectionOnlyAssemblyLoader();
loader.AddHintDirectory(Path.GetDirectoryName(location));
IAssemblyInfo assembly = loader.ReflectionPolicy.LoadAssemblyFrom(location);
var testFrameworkSelector = new TestFrameworkSelector()
{
Filter = testFrameworkHandle => testFrameworkHandle.Id != "MSTestAdapter.TestFramework",
FallbackMode = TestFrameworkFallbackMode.Approximate
};
ITestDriver driver = testFrameworkManager.GetTestDriver(testFrameworkSelector, logger);
TestExplorationOptions testExplorationOptions = new TestExplorationOptions();
ArrayList tests = new ArrayList();
MessageConsumer messageConsumer = new MessageConsumer()
.Handle<TestDiscoveredMessage>(message =>
{
if (message.Test.IsTestCase)
tests.Add(GallioTestElementFactory.CreateTestElement(message.Test, location, projectData));
})
.Handle<AnnotationDiscoveredMessage>(message =>
{
message.Annotation.Log(logger, true);
});
driver.Describe(loader.ReflectionPolicy, new ICodeElementInfo[] { assembly },
testExplorationOptions, messageConsumer, NullProgressMonitor.CreateInstance());
return tests;
}
private void ExploreOrRun(Type driverType, object[] driverArguments,
string assemblyPath, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions,
IMessageSink messageSink, IProgressMonitor progressMonitor)
{
Assembly assembly = LoadAssembly(assemblyPath);
using (var queuedMessageSink = new QueuedMessageSink(messageSink))
{
var testDriver = (DotNetTestDriver)Activator.CreateInstance(driverType, driverArguments);
if (testExecutionOptions == null)
{
testDriver.ExploreAssembly(assembly, testExplorationOptions, queuedMessageSink, progressMonitor);
}
else
{
testDriver.RunAssembly(assembly, testExplorationOptions, testExecutionOptions, queuedMessageSink, progressMonitor);
}
}
}
protected override void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
{
using (progressMonitor.BeginTask("Running tests.", 1))
{
if (testIsolationContext.RequiresSingleThreadedExecution && !testExecutionOptions.SingleThreaded)
{
testExecutionOptions = testExecutionOptions.Copy();
testExecutionOptions.SingleThreaded = true;
}
ForEachDriver(testFrameworkManager.SelectTestFrameworksForFilesImpl(
testFrameworkHandles, testFrameworkFallbackMode, testFrameworkOptions, testPackage.Files),
(driver, items, driverCount) =>
{
TestPackage testPackageForDriver = CreateTestPackageWithFiles(testPackage, items);
driver.Run(testIsolationContext, testPackageForDriver, testExplorationOptions, testExecutionOptions,
messageSink,
progressMonitor.CreateSubProgressMonitor(1.0 / driverCount));
return(false);
});
}
}
/// <summary>
/// Runs tests in an assembly.
/// </summary>
/// <remarks>
/// <para>
/// The default implementation does nothing. Subclasses may override to enable tests
/// to be run. This is required for actually running tests.
/// </para>
/// <para>
/// This method executes within the test host which is most likely a different AppDomain
/// or Process from the test runner itself.
/// </para>
/// </remarks>
/// <param name="assembly">The test assembly, not null.</param>
/// <param name="testExplorationOptions">The test exploration options, not null.</param>
/// <param name="testExecutionOptions">The test execution options, not null.</param>
/// <param name="messageSink">The message sink, not null.</param>
/// <param name="progressMonitor">The progress monitor, not null.</param>
protected virtual void RunAssembly(Assembly assembly, TestExplorationOptions testExplorationOptions,
TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
{
}
/// <summary>
/// Creates a copy of the options.
/// </summary>
/// <returns>The copy.</returns>
public TestExplorationOptions Copy()
{
var copy = new TestExplorationOptions();
copy.properties.AddAll(properties);
return copy;
}
private static List<TestData> FindTests(TestPackage testPackage, ITestRunner runner)
{
var options = new TestExplorationOptions();
testPackage.AddExcludedTestFrameworkId(MsTestFrameworkHandleId);
testPackage.AddExcludedTestFrameworkId(NUnitTestFrameworkHandleId);
var result = runner.Explore(testPackage, options, new ObservableProgressMonitor());
var tests = result.TestModel.AllTests.Where(t => t.IsTestCase).ToList();
return tests;
}
/// <inheritdoc />
public void Explore(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
IMessageSink messageSink, IProgressMonitor progressMonitor)
{
if (testIsolationContext == null)
{
throw new ArgumentNullException("testIsolationContext");
}
if (testPackage == null)
{
throw new ArgumentNullException("testPackage");
}
if (testExplorationOptions == null)
{
throw new ArgumentNullException("testExplorationOptions");
}
if (messageSink == null)
{
throw new ArgumentNullException("messageSink");
}
if (progressMonitor == null)
{
throw new ArgumentNullException("progressMonitor");
}
ExploreImpl(testIsolationContext, testPackage, testExplorationOptions, messageSink, progressMonitor);
}
/// <summary>
/// Runs tests from a test package.
/// </summary>
/// <remarks>
/// <para>
/// The default implementation does nothing. Subclasses may override to enable tests
/// to be run. This is required for actually running tests.
/// </para>
/// </remarks>
/// <param name="testIsolationContext">The test isolation context, not null.</param>
/// <param name="testPackage">The test package, not null.</param>
/// <param name="testExplorationOptions">The test exploration options, not null.</param>
/// <param name="testExecutionOptions">The test execution options, not null.</param>
/// <param name="messageSink">The message sink to receive test exploration and execution messages, not null.</param>
/// <param name="progressMonitor">The progress monitor, not null.</param>
/// <returns>The test report.</returns>
protected virtual void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage,
TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions,
IMessageSink messageSink, IProgressMonitor progressMonitor)
{
}
/// <inheritdoc />
protected override void ExploreImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage,
TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
{
using (progressMonitor.BeginTask("Exploring unsupported tests.", Math.Max(testPackage.Files.Count, 1)))
{
PublishTestModelFromFiles(testPackage.Files, messageSink, progressMonitor);
}
}
private static void ExploreOrRun(TestPackage testPackage, ScriptRuntimeSetup scriptRuntimeSetup, string testDriverScriptPath,
TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions,
IMessageSink messageSink, IProgressMonitor progressMonitor, ILogger logger)
{
using (BufferedLogWriter outputWriter = new BufferedLogWriter(logger, LogSeverity.Info, Encoding.Default),
errorWriter = new BufferedLogWriter(logger, LogSeverity.Error, Encoding.Default))
{
using (var queuedMessageSink = new QueuedMessageSink(messageSink))
{
using (new ConsoleRedirection(outputWriter, errorWriter))
{
var scriptRuntime = new ScriptRuntime(scriptRuntimeSetup);
scriptRuntime.IO.SetInput(Stream.Null, TextReader.Null, Encoding.Default);
scriptRuntime.IO.SetOutput(new TextWriterStream(outputWriter), outputWriter);
scriptRuntime.IO.SetErrorOutput(new TextWriterStream(errorWriter), errorWriter);
try
{
var scriptParameters = new Dictionary<string, object>();
scriptParameters.Add("Verb", testExecutionOptions != null ? "Run" : "Explore");
scriptParameters.Add("TestPackage", testPackage);
scriptParameters.Add("TestExplorationOptions", testExplorationOptions);
scriptParameters.Add("TestExecutionOptions", testExecutionOptions);
scriptParameters.Add("MessageSink", queuedMessageSink);
scriptParameters.Add("ProgressMonitor", progressMonitor);
scriptParameters.Add("Logger", logger);
scriptRuntime.Globals.SetVariable(ScriptParametersVariableName, scriptParameters);
RunScript(scriptRuntime, testDriverScriptPath);
}
finally
{
scriptRuntime.Shutdown();
}
}
}
}
}
private void ExploreOrRun(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor, string taskName)
{
using (progressMonitor.BeginTask(taskName, 1))
{
if (progressMonitor.IsCanceled)
return;
FileInfo testDriverScriptFile = GetTestDriverScriptFile(testPackage);
if (testDriverScriptFile == null)
return;
HostSetup hostSetup = CreateHostSetup(testPackage);
ScriptRuntimeSetup scriptRuntimeSetup = CreateScriptRuntimeSetup(testPackage);
string testDriverScriptPath = testDriverScriptFile.FullName;
var remoteMessageSink = new RemoteMessageSink(messageSink);
var remoteLogger = new RemoteLogger(logger);
using (var remoteProgressMonitor = new RemoteProgressMonitor(progressMonitor.CreateSubProgressMonitor(1)))
{
testIsolationContext.RunIsolatedTask<ExploreOrRunTask>(hostSetup,
(statusMessage) => progressMonitor.SetStatus(statusMessage),
new object[] { testPackage, scriptRuntimeSetup, testDriverScriptPath, testExplorationOptions, testExecutionOptions,
remoteMessageSink, remoteProgressMonitor, remoteLogger });
}
}
}
private void ExploreOrRun(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor, string taskName)
{
using (progressMonitor.BeginTask(taskName, Math.Max(testPackage.Files.Count, 1)))
{
foreach (FileInfo file in testPackage.Files)
{
if (progressMonitor.IsCanceled)
{
return;
}
RemoteMessageSink remoteMessageSink = new RemoteMessageSink(messageSink);
ExploreOrRunAssembly(testIsolationContext, testPackage, testExplorationOptions, testExecutionOptions,
remoteMessageSink, progressMonitor.CreateSubProgressMonitor(1), taskName, file);
}
}
}
private void ExploreOrRunAssembly(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
TestExecutionOptions testExecutionOptions, RemoteMessageSink remoteMessageSink, IProgressMonitor progressMonitor, string taskName,
FileInfo file)
{
using (progressMonitor.BeginTask(taskName, 100))
{
if (progressMonitor.IsCanceled)
{
return;
}
string assemblyPath = file.FullName;
progressMonitor.SetStatus("Getting test assembly metadata.");
AssemblyMetadata assemblyMetadata = AssemblyUtils.GetAssemblyMetadata(assemblyPath, AssemblyMetadataFields.RuntimeVersion);
progressMonitor.Worked(2);
if (progressMonitor.IsCanceled)
{
return;
}
if (assemblyMetadata != null)
{
Type driverType = GetType();
object[] driverArguments = GetRemoteTestDriverArguments();
HostSetup hostSetup = CreateHostSetup(testPackage, assemblyPath, assemblyMetadata);
using (var remoteProgressMonitor = new RemoteProgressMonitor(
progressMonitor.CreateSubProgressMonitor(97)))
{
testIsolationContext.RunIsolatedTask <ExploreOrRunTask>(hostSetup,
(statusMessage) => progressMonitor.SetStatus(statusMessage),
new object[] { driverType, driverArguments, assemblyPath, testExplorationOptions, testExecutionOptions, remoteMessageSink, remoteProgressMonitor });
}
// Record one final work unit after the isolated task has been fully cleaned up.
progressMonitor.SetStatus("");
progressMonitor.Worked(1);
}
}
}
/// <summary>
/// Runs tests from a test package.
/// </summary>
/// <remarks>
/// <para>
/// The default implementation does nothing. Subclasses may override to enable tests
/// to be run. This is required for actually running tests.
/// </para>
/// </remarks>
/// <param name="testIsolationContext">The test isolation context, not null.</param>
/// <param name="testPackage">The test package, not null.</param>
/// <param name="testExplorationOptions">The test exploration options, not null.</param>
/// <param name="testExecutionOptions">The test execution options, not null.</param>
/// <param name="messageSink">The message sink to receive test exploration and execution messages, not null.</param>
/// <param name="progressMonitor">The progress monitor, not null.</param>
/// <returns>The test report.</returns>
protected virtual void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage,
TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions,
IMessageSink messageSink, IProgressMonitor progressMonitor)
{
}
/// <summary>
/// Describes tests via reflection over code elements.
/// </summary>
/// <remarks>
/// <para>
/// The default implementation does nothing. Subclasses may override to enable tests
/// to be described via reflection only. This is required for ReSharper integration.
/// </para>
/// </remarks>
/// <param name="reflectionPolicy">The reflection policy, not null.</param>
/// <param name="codeElements">The enumeration of code elements, usually <see cref="ITypeInfo" />
/// and <see cref="IAssemblyInfo" /> objects, that might contain tests to be described, not null.</param>
/// <param name="testExplorationOptions">The test exploration options, not null.</param>
/// <param name="messageSink">The message sink to receive test exploration messages, not null.</param>
/// <param name="progressMonitor">The progress monitor, not null.</param>
protected virtual void DescribeImpl(IReflectionPolicy reflectionPolicy, IList<ICodeElementInfo> codeElements,
TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
{
}
/// <inheritdoc />
protected override void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage,
TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions,
IMessageSink messageSink, IProgressMonitor progressMonitor)
{
double workItems = Math.Max(testPackage.Files.Count, 1);
using (progressMonitor.BeginTask("Running unsupported tests.", workItems * 2))
{
TestModel testModel = PublishTestModelFromFiles(testPackage.Files, messageSink, progressMonitor);
if (testModel != null)
RunTestModel(testModel, messageSink, progressMonitor, workItems);
}
}
protected override void ExploreImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
{
using (progressMonitor.BeginTask("Exploring tests.", 1))
{
ForEachDriver(testFrameworkManager.SelectTestFrameworksForFilesImpl(
testFrameworkHandles, testFrameworkFallbackMode, testFrameworkOptions, testPackage.Files),
(driver, items, driverCount) =>
{
TestPackage testPackageForDriver = CreateTestPackageWithFiles(testPackage, items);
driver.Explore(testIsolationContext, testPackageForDriver, testExplorationOptions, messageSink,
progressMonitor.CreateSubProgressMonitor(1.0 / driverCount));
return(false);
});
}
}
/// <inheritdoc />
sealed protected override void ExploreImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
{
ExploreOrRun(testIsolationContext, testPackage, testExplorationOptions, null, messageSink, progressMonitor,
"Exploring tests.");
}
/// <summary>
/// Describes tests via reflection over code elements.
/// </summary>
/// <remarks>
/// <para>
/// The default implementation does nothing. Subclasses may override to enable tests
/// to be described via reflection only. This is required for ReSharper integration.
/// </para>
/// </remarks>
/// <param name="reflectionPolicy">The reflection policy, not null.</param>
/// <param name="codeElements">The enumeration of code elements, usually <see cref="ITypeInfo" />
/// and <see cref="IAssemblyInfo" /> objects, that might contain tests to be described, not null.</param>
/// <param name="testExplorationOptions">The test exploration options, not null.</param>
/// <param name="messageSink">The message sink to receive test exploration messages, not null.</param>
/// <param name="progressMonitor">The progress monitor, not null.</param>
protected virtual void DescribeImpl(IReflectionPolicy reflectionPolicy, IList <ICodeElementInfo> codeElements,
TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
{
}
/// <summary>
/// Runs tests in an assembly.
/// </summary>
/// <remarks>
/// <para>
/// The default implementation does nothing. Subclasses may override to enable tests
/// to be run. This is required for actually running tests.
/// </para>
/// <para>
/// This method executes within the test host which is most likely a different AppDomain
/// or Process from the test runner itself.
/// </para>
/// </remarks>
/// <param name="assembly">The test assembly, not null.</param>
/// <param name="testExplorationOptions">The test exploration options, not null.</param>
/// <param name="testExecutionOptions">The test execution options, not null.</param>
/// <param name="messageSink">The message sink, not null.</param>
/// <param name="progressMonitor">The progress monitor, not null.</param>
protected virtual void RunAssembly(Assembly assembly, TestExplorationOptions testExplorationOptions,
TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
{
}
/// <inheritdoc />
public void Describe(IReflectionPolicy reflectionPolicy, IList <ICodeElementInfo> codeElements, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
{
if (reflectionPolicy == null)
{
throw new ArgumentNullException("reflectionPolicy");
}
if (codeElements == null || codeElements.Contains(null))
{
throw new ArgumentNullException("codeElements");
}
if (testExplorationOptions == null)
{
throw new ArgumentNullException("testExplorationOptions");
}
if (messageSink == null)
{
throw new ArgumentNullException("messageSink");
}
if (progressMonitor == null)
{
throw new ArgumentNullException("progressMonitor");
}
DescribeImpl(reflectionPolicy, codeElements, testExplorationOptions, messageSink, progressMonitor);
}
public IEnumerable<AutoTest.TestRunners.Shared.Results.TestResult> Run(RunSettings settings)
{
if (!_isInitialized)
return new AutoTest.TestRunners.Shared.Results.TestResult[] { getNotInitializedResult(settings) };
var tests = settings.Assembly.Tests.ToList();
var members = settings.Assembly.Members.ToList();
var namespaces = settings.Assembly.Namespaces.ToList();
var runAll = namespaces.Count == 0 && members.Count == 0 && tests.Count == 0;
var steps = new List<TestStepData>();
var testResults = new List<AutoTest.TestRunners.Shared.Results.TestResult>();
// Get a test isolation context. Here we want to run tests in the same AppDomain.
var testIsolationProvider = (ITestIsolationProvider)RuntimeAccessor.ServiceLocator.ResolveByComponentId("Gallio.LocalTestIsolationProvider");
var testIsolationOptions = new TestIsolationOptions();
ITestIsolationContext testIsolationContext = testIsolationProvider.CreateContext(testIsolationOptions, _logger);
var testPackage = new TestPackage();
testPackage.AddFile(new FileInfo(settings.Assembly.Assembly));
testPackage.TestFrameworkFallbackMode = TestFrameworkFallbackMode.Strict;
// Create some test exploration options. Nothing interesting for you here, probably.
var testExplorationOptions = new TestExplorationOptions();
var messageSink = new MessageConsumer()
.Handle<TestStepStartedMessage>((message) =>
{
steps.Add(message.Step);
})
.Handle<TestStepFinishedMessage>(message =>
{
var test = steps.FirstOrDefault(x => x.Id.Equals(message.StepId) && x.IsTestCase);
if (test == null)
return;
var fixture = string.Format("{0}.{1}", test.CodeReference.NamespaceName, steps.First(x => x.Id.Equals(test.ParentId)).Name);
testResults.Add(new AutoTest.TestRunners.Shared.Results.TestResult(
"MbUnit",
settings.Assembly.Assembly,
fixture,
message.Result.Duration.TotalMilliseconds,
string.Format("{0}.{1}", fixture, test.Name),
convertState(message.Result.Outcome.Status),
message.Result.Outcome.DisplayName));
});
// Provide a progress monitor.
var logProgressMonitorProvider = new LogProgressMonitorProvider(_logger);
var options = new TestExecutionOptions();
options.FilterSet = new Gallio.Model.Filters.FilterSet<ITestDescriptor>(new OrFilter<ITestDescriptor>(getTestFilter(namespaces, members, tests)));
// Run the tests.
logProgressMonitorProvider.Run((progressMonitor) =>
{
_testDriver.Run(testIsolationContext, testPackage, testExplorationOptions, options, messageSink, progressMonitor);
});
return testResults;
}
/// <inheritdoc />
public Report Run(TestPackage testPackage, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions, IProgressMonitor progressMonitor)
{
if (testPackage == null)
throw new ArgumentNullException("testPackageConfig");
if (testExplorationOptions == null)
throw new ArgumentNullException("testExplorationOptions");
if (testExecutionOptions == null)
throw new ArgumentNullException("testExecutionOptions");
if (progressMonitor == null)
throw new ArgumentNullException("progressMonitor");
ThrowIfDisposed();
if (state != State.Initialized)
throw new InvalidOperationException("The test runner must be initialized before this operation is performed.");
testPackage = testPackage.Copy();
testExplorationOptions = testExplorationOptions.Copy();
testExecutionOptions = testExecutionOptions.Copy();
GenericCollectionUtils.ForEach(testRunnerOptions.Properties, x => testPackage.AddProperty(x.Key, x.Value));
using (progressMonitor.BeginTask("Running the tests.", 10))
{
Stopwatch stopwatch = Stopwatch.StartNew();
Report report = new Report()
{
TestPackage = new TestPackageData(testPackage),
TestModel = new TestModelData(),
TestPackageRun = new TestPackageRun()
{
StartTime = DateTime.Now
}
};
var reportLockBox = new LockBox<Report>(report);
eventDispatcher.NotifyRunStarted(new RunStartedEventArgs(testPackage, testExplorationOptions,
testExecutionOptions, reportLockBox));
bool success;
using (Listener listener = new Listener(eventDispatcher, tappedLogger, reportLockBox))
{
try
{
ITestDriver testDriver = testFrameworkManager.GetTestDriver(
testPackage.CreateTestFrameworkSelector(), tappedLogger);
using (testIsolationContext.BeginBatch(progressMonitor.SetStatus))
{
testDriver.Run(testIsolationContext, testPackage, testExplorationOptions,
testExecutionOptions, listener, progressMonitor.CreateSubProgressMonitor(10));
}
success = true;
}
catch (Exception ex)
{
success = false;
tappedLogger.Log(LogSeverity.Error,
"A fatal exception occurred while running tests. Possible causes include invalid test runner parameters and stack overflows.",
ex);
report.TestModel.Annotations.Add(new AnnotationData(AnnotationType.Error,
CodeLocation.Unknown, CodeReference.Unknown,
"A fatal exception occurred while running tests. See log for details.", null));
}
finally
{
report.TestPackageRun.EndTime = DateTime.Now;
report.TestPackageRun.Statistics.Duration = stopwatch.Elapsed.TotalSeconds;
}
}
eventDispatcher.NotifyRunFinished(new RunFinishedEventArgs(success, report));
return report;
}
}
/// <inheritdoc />
sealed protected override void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
{
ExploreOrRun(testIsolationContext, testPackage, testExplorationOptions, testExecutionOptions, messageSink,
progressMonitor, "Running tests.");
}
private void RunAllTests(IRunContext runContext)
{
ITestRunnerManager runnerManager = RuntimeAccessor.ServiceLocator.Resolve<ITestRunnerManager>();
var runner = runnerManager.CreateTestRunner(StandardTestRunnerFactoryNames.IsolatedAppDomain);
runner.RegisterExtension(new RunContextExtension(runContext));
ILogger logger = new RunContextLogger(runContext);
TestRunnerOptions testRunnerOptions = new TestRunnerOptions();
try
{
RunWithProgressMonitor(delegate(IProgressMonitor progressMonitor)
{
runner.Initialize(testRunnerOptions, logger, progressMonitor);
});
if (isCanceled)
return;
TestPackage testPackage = new TestPackage();
testPackage.AddExcludedTestFrameworkId("MSTestAdapter.TestFramework");
foreach (ITestElement testElement in runContext.RunConfig.TestElements)
{
GallioTestElement gallioTestElement = testElement as GallioTestElement;
if (gallioTestElement != null)
{
testPackage.AddFile(new FileInfo(gallioTestElement.AssemblyPath));
}
}
TestExplorationOptions testExplorationOptions = new TestExplorationOptions();
TestExecutionOptions testExecutionOptions = new TestExecutionOptions();
List<Filter<string>> idFilters = new List<Filter<string>>();
foreach (ITestElement includedTestElement in runContext.RunConfig.TestElements)
{
GallioTestElement gallioTestElement = includedTestElement as GallioTestElement;
if (gallioTestElement != null)
idFilters.Add(new EqualityFilter<string>(gallioTestElement.GallioTestId));
}
testExecutionOptions.FilterSet = new FilterSet<ITestDescriptor>(new IdFilter<ITestDescriptor>(new OrFilter<string>(idFilters)));
RunWithProgressMonitor(delegate(IProgressMonitor progressMonitor)
{
runner.Run(testPackage, testExplorationOptions, testExecutionOptions, progressMonitor);
});
}
finally
{
runner.Dispose(NullProgressMonitor.CreateInstance());
}
}
TestModel PopulateTestTree(Assembly assembly)
{
TestModel testModel = new TestModel();
var testFrameworkManager = RuntimeAccessor.ServiceLocator.Resolve<ITestFrameworkManager>();
var logger = new MarkupStreamLogger(TestLog.Default);
var testFrameworkSelector = new TestFrameworkSelector()
{
Filter = testFrameworkHandle => testFrameworkHandle.Id == TestFrameworkHandle.Id,
FallbackMode = TestFrameworkFallbackMode.Strict
};
ITestDriver testDriver = testFrameworkManager.GetTestDriver(testFrameworkSelector, logger);
var testIsolationProvider = (ITestIsolationProvider)RuntimeAccessor.ServiceLocator.ResolveByComponentId("Gallio.LocalTestIsolationProvider");
var testIsolationOptions = new TestIsolationOptions();
using (ITestIsolationContext testIsolationContext = testIsolationProvider.CreateContext(testIsolationOptions, logger))
{
var testPackage = new TestPackage();
testPackage.AddFile(new FileInfo(AssemblyUtils.GetFriendlyAssemblyCodeBase(assembly)));
var testExplorationOptions = new TestExplorationOptions();
var messageSink = TestModelSerializer.CreateMessageSinkToPopulateTestModel(testModel);
new LogProgressMonitorProvider(logger).Run(progressMonitor =>
{
testDriver.Explore(testIsolationContext, testPackage, testExplorationOptions,
messageSink, progressMonitor);
});
}
return testModel;
}
/// <inheritdoc />
protected override sealed void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
{
ExploreOrRun(testIsolationContext, testPackage, testExplorationOptions, testExecutionOptions, messageSink,
progressMonitor, "Running tests.");
}
protected override void DescribeImpl(IReflectionPolicy reflectionPolicy, IList <ICodeElementInfo> codeElements, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, Gallio.Runtime.ProgressMonitoring.IProgressMonitor progressMonitor)
{
using (progressMonitor.BeginTask("Describing tests.", 1))
{
ForEachDriver(testFrameworkManager.SelectTestFrameworksForCodeElementsImpl(
testFrameworkHandles, testFrameworkFallbackMode, testFrameworkOptions, codeElements),
(driver, items, driverCount) =>
{
driver.Describe(reflectionPolicy, items, testExplorationOptions, messageSink,
progressMonitor.CreateSubProgressMonitor(1.0 / driverCount));
return(false);
});
}
}
private List<Test> getTests(string assembly)
{
var testIsolationProvider = (ITestIsolationProvider)RuntimeAccessor.ServiceLocator.ResolveByComponentId("Gallio.LocalTestIsolationProvider");
var testIsolationOptions = new TestIsolationOptions();
ITestIsolationContext testIsolationContext = testIsolationProvider.CreateContext(testIsolationOptions, _logger);
// Create a test package.
// You can set a whole bunch of options here.
var testPackage = new TestPackage();
testPackage.AddFile(new FileInfo(assembly));
testPackage.TestFrameworkFallbackMode = TestFrameworkFallbackMode.Strict;
var testExplorationOptions = new TestExplorationOptions();
// This query you can answer by exploring tests and looking at their metadata for "TestsOn".
// You can explore tests using the Explore method of TestDriver. It sends a bunch of
// messages to an IMessageSink just like Run. Just scan these messages for tests that
// match the pattern you like.
// Alternately, you can build a TestModel once up front and traverse the resulting test tree
// to find what you need. The Explore method of testDriver is just like Run, except that
// it does not run the tests.
// TestModelSerializer is a little helper we can use to build up a TestModel from test messages on the fly.
// The TestModel is just a tree of test metadata. Pretty straightforward.
TestModel testModel = new TestModel();
IMessageSink messageSink = TestModelSerializer.CreateMessageSinkToPopulateTestModel(testModel);
var logProgressMonitorProvider = new LogProgressMonitorProvider(_logger);
logProgressMonitorProvider.Run((progressMonitor) =>
{
_testDriver.Explore(testIsolationContext, testPackage, testExplorationOptions, messageSink, progressMonitor);
});
return testModel.AllTests.Where(x => x.IsTestCase).ToList();
}
private void ExploreOrRun(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor, string taskName)
{
using (progressMonitor.BeginTask(taskName, Math.Max(testPackage.Files.Count, 1)))
{
foreach (FileInfo file in testPackage.Files)
{
if (progressMonitor.IsCanceled)
return;
RemoteMessageSink remoteMessageSink = new RemoteMessageSink(messageSink);
ExploreOrRunAssembly(testIsolationContext, testPackage, testExplorationOptions, testExecutionOptions,
remoteMessageSink, progressMonitor.CreateSubProgressMonitor(1), taskName, file);
}
}
}
private void Describe(IReflectionPolicy reflectionPolicy, IList<ICodeElementInfo> codeElements, ConsumerAdapter consumerAdapter)
{
var testDriver = CreateTestDriver();
var testExplorationOptions = new TestExplorationOptions();
testDriver.Describe(reflectionPolicy, codeElements, testExplorationOptions, consumerAdapter,
NullProgressMonitor.CreateInstance());
}
private void ExploreOrRunAssembly(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
TestExecutionOptions testExecutionOptions, RemoteMessageSink remoteMessageSink, IProgressMonitor progressMonitor, string taskName,
FileInfo file)
{
using (progressMonitor.BeginTask(taskName, 100))
{
if (progressMonitor.IsCanceled)
return;
string assemblyPath = file.FullName;
progressMonitor.SetStatus("Getting test assembly metadata.");
AssemblyMetadata assemblyMetadata = AssemblyUtils.GetAssemblyMetadata(assemblyPath, AssemblyMetadataFields.RuntimeVersion);
progressMonitor.Worked(2);
if (progressMonitor.IsCanceled)
return;
if (assemblyMetadata != null)
{
Type driverType = GetType();
object[] driverArguments = GetRemoteTestDriverArguments();
HostSetup hostSetup = CreateHostSetup(testPackage, assemblyPath, assemblyMetadata);
using (var remoteProgressMonitor = new RemoteProgressMonitor(
progressMonitor.CreateSubProgressMonitor(97)))
{
testIsolationContext.RunIsolatedTask<ExploreOrRunTask>(hostSetup,
(statusMessage) => progressMonitor.SetStatus(statusMessage),
new object[] { driverType, driverArguments, assemblyPath, testExplorationOptions, testExecutionOptions, remoteMessageSink, remoteProgressMonitor });
}
// Record one final work unit after the isolated task has been fully cleaned up.
progressMonitor.SetStatus("");
progressMonitor.Worked(1);
}
}
}
private void ExploreOrRun(Type driverType, object[] driverArguments,
string assemblyPath, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions,
IMessageSink messageSink, IProgressMonitor progressMonitor)
{
Assembly assembly = LoadAssembly(assemblyPath);
using (var queuedMessageSink = new QueuedMessageSink(messageSink))
{
var testDriver = (DotNetTestDriver) Activator.CreateInstance(driverType, driverArguments);
if (testExecutionOptions == null)
testDriver.ExploreAssembly(assembly, testExplorationOptions, queuedMessageSink, progressMonitor);
else
testDriver.RunAssembly(assembly, testExplorationOptions, testExecutionOptions, queuedMessageSink, progressMonitor);
}
}
private FacadeTaskResult RunTests()
{
var logger = new FacadeLoggerWrapper(facadeLogger);
var runner = TestRunnerUtils.CreateTestRunnerByName(StandardTestRunnerFactoryNames.IsolatedAppDomain);
// Set parameters.
var testPackage = new TestPackage();
foreach (var assemblyLocation in assemblyLocations)
testPackage.AddFile(new FileInfo(assemblyLocation));
testPackage.ShadowCopy = facadeTaskExecutorConfiguration.ShadowCopy;
if (facadeTaskExecutorConfiguration.AssemblyFolder != null)
{
testPackage.ApplicationBaseDirectory = new DirectoryInfo(facadeTaskExecutorConfiguration.AssemblyFolder);
testPackage.WorkingDirectory = new DirectoryInfo(facadeTaskExecutorConfiguration.AssemblyFolder);
}
var testRunnerOptions = new TestRunnerOptions();
var testExplorationOptions = new TestExplorationOptions();
var filters = GenericCollectionUtils.ConvertAllToArray<string, Filter<string>>(explicitTestIds,
testId => new EqualityFilter<string>(testId));
var filterSet = new FilterSet<ITestDescriptor>(new IdFilter<ITestDescriptor>(new OrFilter<string>(filters)));
var testExecutionOptions = new TestExecutionOptions { FilterSet = filterSet };
// Install the listeners.
runner.Events.TestStepStarted += TestStepStarted;
runner.Events.TestStepFinished += TestStepFinished;
runner.Events.TestStepLifecyclePhaseChanged += TestStepLifecyclePhaseChanged;
// Run the tests.
try
{
try
{
runner.Initialize(testRunnerOptions, logger, CreateProgressMonitor());
Report report = runner.Run(testPackage, testExplorationOptions, testExecutionOptions, CreateProgressMonitor());
if (sessionId != null)
SessionCache.SaveSerializedReport(sessionId, report);
return FacadeTaskResult.Success;
}
catch (Exception ex)
{
if (sessionId != null)
SessionCache.ClearSerializedReport(sessionId);
logger.Log(LogSeverity.Error, "A fatal exception occurred during test execution.", ex);
return FacadeTaskResult.Exception;
}
finally
{
SubmitFailureForRemainingPendingTasks();
}
}
finally
{
runner.Dispose(CreateProgressMonitor());
}
}