/// <inheritdoc />
public void Explore(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
IMessageSink messageSink, IProgressMonitor progressMonitor)
{
if (testIsolationContext == null)
{
throw new ArgumentNullException("testIsolationContext");
}
if (testPackage == null)
{
throw new ArgumentNullException("testPackage");
}
if (testExplorationOptions == null)
{
throw new ArgumentNullException("testExplorationOptions");
}
if (messageSink == null)
{
throw new ArgumentNullException("messageSink");
}
if (progressMonitor == null)
{
throw new ArgumentNullException("progressMonitor");
}
ExploreImpl(testIsolationContext, testPackage, testExplorationOptions, messageSink, progressMonitor);
}
protected TestModel PopulateTestTree(Assembly assembly)
{
TestModel testModel = new TestModel();
var testFrameworkManager = RuntimeAccessor.ServiceLocator.Resolve <ITestFrameworkManager>();
var logger = new MarkupStreamLogger(TestLog.Default);
var testFrameworkSelector = new TestFrameworkSelector()
{
Filter = testFrameworkHandle => testFrameworkHandle.Id == TestFrameworkHandle.Id,
FallbackMode = TestFrameworkFallbackMode.Strict
};
ITestDriver testDriver = testFrameworkManager.GetTestDriver(testFrameworkSelector, logger);
var testIsolationProvider = (ITestIsolationProvider)RuntimeAccessor.ServiceLocator.ResolveByComponentId("Gallio.LocalTestIsolationProvider");
var testIsolationOptions = new TestIsolationOptions();
using (ITestIsolationContext testIsolationContext = testIsolationProvider.CreateContext(testIsolationOptions, logger))
{
var testPackage = new TestPackage();
testPackage.AddFile(new FileInfo(AssemblyUtils.GetFriendlyAssemblyCodeBase(assembly)));
var testExplorationOptions = new TestExplorationOptions();
var messageSink = TestModelSerializer.CreateMessageSinkToPopulateTestModel(testModel);
new LogProgressMonitorProvider(logger).Run(progressMonitor =>
{
testDriver.Explore(testIsolationContext, testPackage, testExplorationOptions,
messageSink, progressMonitor);
});
}
return(testModel);
}
private void ExploreOrRun(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor, string taskName)
{
using (progressMonitor.BeginTask(taskName, 1))
{
if (progressMonitor.IsCanceled)
{
return;
}
FileInfo testDriverScriptFile = GetTestDriverScriptFile(testPackage);
if (testDriverScriptFile == null)
{
return;
}
HostSetup hostSetup = CreateHostSetup(testPackage);
ScriptRuntimeSetup scriptRuntimeSetup = CreateScriptRuntimeSetup(testPackage);
string testDriverScriptPath = testDriverScriptFile.FullName;
var remoteMessageSink = new RemoteMessageSink(messageSink);
var remoteLogger = new RemoteLogger(logger);
using (var remoteProgressMonitor = new RemoteProgressMonitor(progressMonitor.CreateSubProgressMonitor(1)))
{
testIsolationContext.RunIsolatedTask <ExploreOrRunTask>(hostSetup,
(statusMessage) => progressMonitor.SetStatus(statusMessage),
new object[] { testPackage, scriptRuntimeSetup, testDriverScriptPath, testExplorationOptions, testExecutionOptions,
remoteMessageSink, remoteProgressMonitor, remoteLogger });
}
}
}
/// <inheritdoc />
protected override void ExploreImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage,
TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
{
using (progressMonitor.BeginTask("Exploring unsupported tests.", Math.Max(testPackage.Files.Count, 1)))
{
PublishTestModelFromFiles(testPackage.Files, messageSink, progressMonitor);
}
}
static void TestTypeByNameWithNoInvalidAssembly(ITestIsolationContext context)
{
context.AssemblyLoad("HarmonyTestsDummyAssemblyC");
Assert.NotNull(AccessTools.TypeByName(typeof(Harmony).FullName));
Assert.NotNull(AccessTools.TypeByName(typeof(Test_AccessTools).FullName));
Assert.Null(AccessTools.TypeByName("HarmonyTestsDummyAssemblyA.Class1"));
Assert.Null(AccessTools.TypeByName("HarmonyTestsDummyAssemblyB.Class1"));
Assert.Null(AccessTools.TypeByName("HarmonyTestsDummyAssemblyB.Class2"));
Assert.NotNull(AccessTools.TypeByName("HarmonyTestsDummyAssemblyC.Class1"));
Assert.Null(AccessTools.TypeByName("IAmALittleTeaPot.ShortAndStout"));
}
public void SetUp()
{
logger = MockRepository.GenerateStub<ILogger>();
driver = new IronRubyTestDriver(logger);
var testIsolationProvider = (ITestIsolationProvider)RuntimeAccessor.ServiceLocator.ResolveByComponentId("Gallio.LocalTestIsolationProvider");
testIsolationContext = testIsolationProvider.CreateContext(new TestIsolationOptions(), logger);
testPackage = new TestPackage();
testExplorationOptions = new TestExplorationOptions();
testExecutionOptions = new TestExecutionOptions();
messageSink = MockRepository.GenerateStub<IMessageSink>();
progressMonitor = NullProgressMonitor.CreateInstance();
}
public void SetUp()
{
logger = MockRepository.GenerateStub <ILogger>();
driver = new IronRubyTestDriver(logger);
var testIsolationProvider = (ITestIsolationProvider)RuntimeAccessor.ServiceLocator.ResolveByComponentId("Gallio.LocalTestIsolationProvider");
testIsolationContext = testIsolationProvider.CreateContext(new TestIsolationOptions(), logger);
testPackage = new TestPackage();
testExplorationOptions = new TestExplorationOptions();
testExecutionOptions = new TestExecutionOptions();
messageSink = MockRepository.GenerateStub <IMessageSink>();
progressMonitor = NullProgressMonitor.CreateInstance();
}
protected override void ExploreImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
{
using (progressMonitor.BeginTask("Exploring tests.", 1))
{
ForEachDriver(testFrameworkManager.SelectTestFrameworksForFilesImpl(
testFrameworkHandles, testFrameworkFallbackMode, testFrameworkOptions, testPackage.Files),
(driver, items, driverCount) =>
{
TestPackage testPackageForDriver = CreateTestPackageWithFiles(testPackage, items);
driver.Explore(testIsolationContext, testPackageForDriver, testExplorationOptions, messageSink,
progressMonitor.CreateSubProgressMonitor(1.0 / driverCount));
return(false);
});
}
}
/// <inheritdoc />
protected override void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage,
TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions,
IMessageSink messageSink, IProgressMonitor progressMonitor)
{
double workItems = Math.Max(testPackage.Files.Count, 1);
using (progressMonitor.BeginTask("Running unsupported tests.", workItems * 2))
{
TestModel testModel = PublishTestModelFromFiles(testPackage.Files, messageSink, progressMonitor);
if (testModel != null)
{
RunTestModel(testModel, messageSink, progressMonitor, workItems);
}
}
}
/// <inheritdoc />
public void Explore(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
IMessageSink messageSink, IProgressMonitor progressMonitor)
{
if (testIsolationContext == null)
throw new ArgumentNullException("testIsolationContext");
if (testPackage == null)
throw new ArgumentNullException("testPackage");
if (testExplorationOptions == null)
throw new ArgumentNullException("testExplorationOptions");
if (messageSink == null)
throw new ArgumentNullException("messageSink");
if (progressMonitor == null)
throw new ArgumentNullException("progressMonitor");
ExploreImpl(testIsolationContext, testPackage, testExplorationOptions, messageSink, progressMonitor);
}
private void ExploreOrRun(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor, string taskName)
{
using (progressMonitor.BeginTask(taskName, Math.Max(testPackage.Files.Count, 1)))
{
foreach (FileInfo file in testPackage.Files)
{
if (progressMonitor.IsCanceled)
{
return;
}
RemoteMessageSink remoteMessageSink = new RemoteMessageSink(messageSink);
ExploreOrRunAssembly(testIsolationContext, testPackage, testExplorationOptions, testExecutionOptions,
remoteMessageSink, progressMonitor.CreateSubProgressMonitor(1), taskName, file);
}
}
}
static void CreateTestDummyAssemblies(ITestIsolationContext context)
{
var dummyAssemblyA = DefineAssembly("HarmonyTestsDummyAssemblyA",
moduleBuilder => moduleBuilder.DefineType("HarmonyTestsDummyAssemblyA.Class1", TypeAttributes.Public));
// Explicitly NOT saving HarmonyTestsDummyAssemblyA.
var dummyAssemblyB = DefineAssembly("HarmonyTestsDummyAssemblyB",
moduleBuilder => moduleBuilder.DefineType("HarmonyTestsDummyAssemblyB.Class1", TypeAttributes.Public,
parent: dummyAssemblyA.GetType("HarmonyTestsDummyAssemblyA.Class1")),
moduleBuilder => moduleBuilder.DefineType("HarmonyTestsDummyAssemblyB.Class2", TypeAttributes.Public));
// HarmonyTestsDummyAssemblyB, if loaded, becomes an invalid assembly due to missing HarmonyTestsDummyAssemblyA.
SaveAssembly(dummyAssemblyB);
// HarmonyTestsDummyAssemblyC is just another (valid) assembly to be loaded after HarmonyTestsDummyAssemblyB.
var dummyAssemblyC = DefineAssembly("HarmonyTestsDummyAssemblyC",
moduleBuilder => moduleBuilder.DefineType("HarmonyTestsDummyAssemblyC.Class1", TypeAttributes.Public));
SaveAssembly(dummyAssemblyC);
}
/// <inheritdoc />
public void Dispose(IProgressMonitor progressMonitor)
{
if (progressMonitor == null)
{
throw new ArgumentNullException("progressMonitor");
}
if (state == State.Disposed)
{
return;
}
using (progressMonitor.BeginTask("Disposing the test runner.", 10))
{
bool success;
try
{
eventDispatcher.NotifyDisposeStarted(new DisposeStartedEventArgs());
if (testIsolationContext != null)
{
progressMonitor.SetStatus("Disposing the test isolation context.");
testIsolationContext.Dispose();
testIsolationContext = null;
}
progressMonitor.Worked(10);
success = true;
}
catch (Exception ex)
{
if (tappedLogger != null)
{
tappedLogger.Log(LogSeverity.Warning, "An exception occurred while disposing the test isolation context. This may indicate that the test isolation context previously encountered another fault from which it could not recover.", ex);
}
success = false;
}
state = State.Disposed;
eventDispatcher.NotifyDisposeFinished(new DisposeFinishedEventArgs(success));
UnhandledExceptionPolicy.ReportUnhandledException -= OnUnhandledException;
}
}
private void ExploreOrRunAssembly(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
TestExecutionOptions testExecutionOptions, RemoteMessageSink remoteMessageSink, IProgressMonitor progressMonitor, string taskName,
FileInfo file)
{
using (progressMonitor.BeginTask(taskName, 100))
{
if (progressMonitor.IsCanceled)
{
return;
}
string assemblyPath = file.FullName;
progressMonitor.SetStatus("Getting test assembly metadata.");
AssemblyMetadata assemblyMetadata = AssemblyUtils.GetAssemblyMetadata(assemblyPath, AssemblyMetadataFields.RuntimeVersion);
progressMonitor.Worked(2);
if (progressMonitor.IsCanceled)
{
return;
}
if (assemblyMetadata != null)
{
Type driverType = GetType();
object[] driverArguments = GetRemoteTestDriverArguments();
HostSetup hostSetup = CreateHostSetup(testPackage, assemblyPath, assemblyMetadata);
using (var remoteProgressMonitor = new RemoteProgressMonitor(
progressMonitor.CreateSubProgressMonitor(97)))
{
testIsolationContext.RunIsolatedTask <ExploreOrRunTask>(hostSetup,
(statusMessage) => progressMonitor.SetStatus(statusMessage),
new object[] { driverType, driverArguments, assemblyPath, testExplorationOptions, testExecutionOptions, remoteMessageSink, remoteProgressMonitor });
}
// Record one final work unit after the isolated task has been fully cleaned up.
progressMonitor.SetStatus("");
progressMonitor.Worked(1);
}
}
}
protected override void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
{
using (progressMonitor.BeginTask("Running tests.", 1))
{
if (testIsolationContext.RequiresSingleThreadedExecution && !testExecutionOptions.SingleThreaded)
{
testExecutionOptions = testExecutionOptions.Copy();
testExecutionOptions.SingleThreaded = true;
}
ForEachDriver(testFrameworkManager.SelectTestFrameworksForFilesImpl(
testFrameworkHandles, testFrameworkFallbackMode, testFrameworkOptions, testPackage.Files),
(driver, items, driverCount) =>
{
TestPackage testPackageForDriver = CreateTestPackageWithFiles(testPackage, items);
driver.Run(testIsolationContext, testPackageForDriver, testExplorationOptions, testExecutionOptions,
messageSink,
progressMonitor.CreateSubProgressMonitor(1.0 / driverCount));
return(false);
});
}
}
private List <Test> getTests(string assembly)
{
var testIsolationProvider = (ITestIsolationProvider)RuntimeAccessor.ServiceLocator.ResolveByComponentId("Gallio.LocalTestIsolationProvider");
var testIsolationOptions = new TestIsolationOptions();
ITestIsolationContext testIsolationContext = testIsolationProvider.CreateContext(testIsolationOptions, _logger);
// Create a test package.
// You can set a whole bunch of options here.
var testPackage = new TestPackage();
testPackage.AddFile(new FileInfo(assembly));
testPackage.TestFrameworkFallbackMode = TestFrameworkFallbackMode.Strict;
var testExplorationOptions = new TestExplorationOptions();
// This query you can answer by exploring tests and looking at their metadata for "TestsOn".
// You can explore tests using the Explore method of TestDriver. It sends a bunch of
// messages to an IMessageSink just like Run. Just scan these messages for tests that
// match the pattern you like.
// Alternately, you can build a TestModel once up front and traverse the resulting test tree
// to find what you need. The Explore method of testDriver is just like Run, except that
// it does not run the tests.
// TestModelSerializer is a little helper we can use to build up a TestModel from test messages on the fly.
// The TestModel is just a tree of test metadata. Pretty straightforward.
TestModel testModel = new TestModel();
IMessageSink messageSink = TestModelSerializer.CreateMessageSinkToPopulateTestModel(testModel);
var logProgressMonitorProvider = new LogProgressMonitorProvider(_logger);
logProgressMonitorProvider.Run((progressMonitor) =>
{
_testDriver.Explore(testIsolationContext, testPackage, testExplorationOptions, messageSink, progressMonitor);
});
return(testModel.AllTests.Where(x => x.IsTestCase).ToList());
}
private void ExploreOrRun <TTask>(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor, string taskName)
where TTask : AbstractTask, new()
{
double totalWorkUnits = Math.Max(testPackage.Files.Count, 1);
using (progressMonitor.BeginTask(taskName, totalWorkUnits))
{
var remoteMessageSink = new RemoteMessageSink(messageSink);
var remoteLogger = new RemoteLogger(logger);
if (progressMonitor.IsCanceled)
{
return;
}
using (var remoteProgressMonitor = new RemoteProgressMonitor(progressMonitor))
{
foreach (FileInfo file in testPackage.Files)
{
HostSetup hostSetup = CreateHostSetup(testPackage, file);
testIsolationContext.RunIsolatedTask <TTask>(hostSetup, progressMonitor.SetStatus, new object[]
{
testPackage,
testExplorationOptions,
testExecutionOptions,
remoteMessageSink,
remoteProgressMonitor,
remoteLogger,
file,
});
}
remoteProgressMonitor.Worked(1);
}
}
}
static void TestTypeByNameWithInvalidAssembly(ITestIsolationContext context)
{
// HarmonyTestsDummyAssemblyB has a dependency on HarmonyTestsDummyAssemblyA, but we've ensured that
// HarmonyTestsDummyAssemblyA.dll is NOT available (i.e. not in HarmonyTests output dir).
context.AssemblyLoad("HarmonyTestsDummyAssemblyB");
context.AssemblyLoad("HarmonyTestsDummyAssemblyC");
// Even if 0Harmony.dll isn't loaded yet and thus would be automatically loaded after the invalid assemblies,
// TypeByName tries Type.GetType first, which always works for a type in the executing assembly (0Harmony.dll).
Assert.NotNull(AccessTools.TypeByName(typeof(Harmony).FullName));
// The current executing assembly (HarmonyTests.dll) was definitely already loaded before above loads.
Assert.NotNull(AccessTools.TypeByName(typeof(Test_AccessTools).FullName));
// HarmonyTestsDummyAssemblyA is explicitly missing, so it's the same as the unknown type case - see below.
Assert.Null(AccessTools.TypeByName("HarmonyTestsDummyAssemblyA.Class1"));
// HarmonyTestsDummyAssemblyB.GetTypes() should throw ReflectionTypeLoadException due to missing HarmonyTestsDummyAssemblyA,
// but this is caught and returns successfully loaded types.
// HarmonyTestsDummyAssemblyB.Class1 depends on HarmonyTestsDummyAssemblyA, so it's not loaded successfully.
Assert.Null(AccessTools.TypeByName("HarmonyTestsDummyAssemblyB.Class1"));
// HarmonyTestsDummyAssemblyB.Class2 doesn't depend on HarmonyTestsDummyAssemblyA, so it's loaded successfully.
Assert.NotNull(AccessTools.TypeByName("HarmonyTestsDummyAssemblyB.Class2"));
// TypeByName's search should find HarmonyTestsDummyAssemblyB before HarmonyTestsDummyAssemblyC, but this is fine.
Assert.NotNull(AccessTools.TypeByName("HarmonyTestsDummyAssemblyC.Class1"));
// TypeByName's search for an unknown type should always find HarmonyTestsDummyAssemblyB first, which is again fine.
Assert.Null(AccessTools.TypeByName("IAmALittleTeaPot.ShortAndStout"));
}
/// <inheritdoc />
protected sealed override void ExploreImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
{
ExploreOrRun <ExploreTask>(testIsolationContext, testPackage, testExplorationOptions, null, messageSink, progressMonitor, "Exploring MbUnitCpp tests.");
}
private void ExploreOrRun(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor, string taskName)
{
using (progressMonitor.BeginTask(taskName, Math.Max(testPackage.Files.Count, 1)))
{
foreach (FileInfo file in testPackage.Files)
{
if (progressMonitor.IsCanceled)
return;
RemoteMessageSink remoteMessageSink = new RemoteMessageSink(messageSink);
ExploreOrRunAssembly(testIsolationContext, testPackage, testExplorationOptions, testExecutionOptions,
remoteMessageSink, progressMonitor.CreateSubProgressMonitor(1), taskName, file);
}
}
}
/// <inheritdoc />
public void Dispose(IProgressMonitor progressMonitor)
{
if (progressMonitor == null)
throw new ArgumentNullException("progressMonitor");
if (state == State.Disposed)
return;
using (progressMonitor.BeginTask("Disposing the test runner.", 10))
{
bool success;
try
{
eventDispatcher.NotifyDisposeStarted(new DisposeStartedEventArgs());
if (testIsolationContext != null)
{
progressMonitor.SetStatus("Disposing the test isolation context.");
testIsolationContext.Dispose();
testIsolationContext = null;
}
progressMonitor.Worked(10);
success = true;
}
catch (Exception ex)
{
if (tappedLogger != null)
tappedLogger.Log(LogSeverity.Warning, "An exception occurred while disposing the test isolation context. This may indicate that the test isolation context previously encountered another fault from which it could not recover.", ex);
success = false;
}
state = State.Disposed;
eventDispatcher.NotifyDisposeFinished(new DisposeFinishedEventArgs(success));
UnhandledExceptionPolicy.ReportUnhandledException -= OnUnhandledException;
}
}
/// <inheritdoc />
public void Initialize(TestRunnerOptions testRunnerOptions, ILogger logger, IProgressMonitor progressMonitor)
{
if (testRunnerOptions == null)
throw new ArgumentNullException("testRunnerOptions");
if (logger == null)
throw new ArgumentNullException("logger");
if (progressMonitor == null)
throw new ArgumentNullException("progressMonitor");
ThrowIfDisposed();
if (state != State.Created)
throw new InvalidOperationException("The test runner has already been initialized.");
testRunnerOptions = testRunnerOptions.Copy();
this.testRunnerOptions = testRunnerOptions;
tappedLogger = new TappedLogger(this, logger);
int extensionCount = extensions.Count;
using (progressMonitor.BeginTask("Initializing the test runner.", 1 + extensionCount))
{
foreach (ITestRunnerExtension extension in extensions)
{
string extensionName = extension.GetType().Name; // TODO: improve me
progressMonitor.SetStatus(String.Format("Installing extension '{0}'.", extensionName));
try
{
// Note: We don't pass the tapped logger to the extensions because the
// extensions frequently write to the console a bunch of information we
// already have represented in the report. We are more interested in what
// the test driver has to tell us.
extension.Install(eventDispatcher, logger);
progressMonitor.Worked(1);
}
catch (Exception ex)
{
throw new RunnerException(String.Format("Failed to install extension '{0}'.", extensionName), ex);
}
progressMonitor.SetStatus("");
}
try
{
UnhandledExceptionPolicy.ReportUnhandledException += OnUnhandledException;
eventDispatcher.NotifyInitializeStarted(new InitializeStartedEventArgs(testRunnerOptions));
progressMonitor.SetStatus("Initializing the test isolation context.");
TestIsolationOptions testIsolationOptions = new TestIsolationOptions();
GenericCollectionUtils.ForEach(testRunnerOptions.Properties, x => testIsolationOptions.AddProperty(x.Key, x.Value));
testIsolationContext = testIsolationProvider.CreateContext(testIsolationOptions, tappedLogger);
progressMonitor.Worked(1);
}
catch (Exception ex)
{
eventDispatcher.NotifyInitializeFinished(new InitializeFinishedEventArgs(false));
UnhandledExceptionPolicy.ReportUnhandledException -= OnUnhandledException;
throw new RunnerException("A fatal exception occurred while initializing the test isolation context.", ex);
}
state = State.Initialized;
eventDispatcher.NotifyInitializeFinished(new InitializeFinishedEventArgs(true));
}
}
/// <summary>
/// Runs tests from a test package.
/// </summary>
/// <remarks>
/// <para>
/// The default implementation does nothing. Subclasses may override to enable tests
/// to be run. This is required for actually running tests.
/// </para>
/// </remarks>
/// <param name="testIsolationContext">The test isolation context, not null.</param>
/// <param name="testPackage">The test package, not null.</param>
/// <param name="testExplorationOptions">The test exploration options, not null.</param>
/// <param name="testExecutionOptions">The test execution options, not null.</param>
/// <param name="messageSink">The message sink to receive test exploration and execution messages, not null.</param>
/// <param name="progressMonitor">The progress monitor, not null.</param>
/// <returns>The test report.</returns>
protected virtual void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage,
TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions,
IMessageSink messageSink, IProgressMonitor progressMonitor)
{
}
/// <inheritdoc />
public void Initialize(TestRunnerOptions testRunnerOptions, ILogger logger, IProgressMonitor progressMonitor)
{
if (testRunnerOptions == null)
{
throw new ArgumentNullException("testRunnerOptions");
}
if (logger == null)
{
throw new ArgumentNullException("logger");
}
if (progressMonitor == null)
{
throw new ArgumentNullException("progressMonitor");
}
ThrowIfDisposed();
if (state != State.Created)
{
throw new InvalidOperationException("The test runner has already been initialized.");
}
testRunnerOptions = testRunnerOptions.Copy();
this.testRunnerOptions = testRunnerOptions;
tappedLogger = new TappedLogger(this, logger);
int extensionCount = extensions.Count;
using (progressMonitor.BeginTask("Initializing the test runner.", 1 + extensionCount))
{
foreach (ITestRunnerExtension extension in extensions)
{
string extensionName = extension.GetType().Name; // TODO: improve me
progressMonitor.SetStatus(String.Format("Installing extension '{0}'.", extensionName));
try
{
// Note: We don't pass the tapped logger to the extensions because the
// extensions frequently write to the console a bunch of information we
// already have represented in the report. We are more interested in what
// the test driver has to tell us.
extension.Install(eventDispatcher, logger);
progressMonitor.Worked(1);
}
catch (Exception ex)
{
throw new RunnerException(String.Format("Failed to install extension '{0}'.", extensionName), ex);
}
progressMonitor.SetStatus("");
}
try
{
UnhandledExceptionPolicy.ReportUnhandledException += OnUnhandledException;
eventDispatcher.NotifyInitializeStarted(new InitializeStartedEventArgs(testRunnerOptions));
progressMonitor.SetStatus("Initializing the test isolation context.");
TestIsolationOptions testIsolationOptions = new TestIsolationOptions();
GenericCollectionUtils.ForEach(testRunnerOptions.Properties, x => testIsolationOptions.AddProperty(x.Key, x.Value));
testIsolationContext = testIsolationProvider.CreateContext(testIsolationOptions, tappedLogger);
progressMonitor.Worked(1);
}
catch (Exception ex)
{
eventDispatcher.NotifyInitializeFinished(new InitializeFinishedEventArgs(false));
UnhandledExceptionPolicy.ReportUnhandledException -= OnUnhandledException;
throw new RunnerException("A fatal exception occurred while initializing the test isolation context.", ex);
}
state = State.Initialized;
eventDispatcher.NotifyInitializeFinished(new InitializeFinishedEventArgs(true));
}
}
/// <summary>
/// Runs tests from a test package.
/// </summary>
/// <remarks>
/// <para>
/// The default implementation does nothing. Subclasses may override to enable tests
/// to be run. This is required for actually running tests.
/// </para>
/// </remarks>
/// <param name="testIsolationContext">The test isolation context, not null.</param>
/// <param name="testPackage">The test package, not null.</param>
/// <param name="testExplorationOptions">The test exploration options, not null.</param>
/// <param name="testExecutionOptions">The test execution options, not null.</param>
/// <param name="messageSink">The message sink to receive test exploration and execution messages, not null.</param>
/// <param name="progressMonitor">The progress monitor, not null.</param>
/// <returns>The test report.</returns>
protected virtual void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage,
TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions,
IMessageSink messageSink, IProgressMonitor progressMonitor)
{
}
private void ExploreOrRun(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor, string taskName)
{
using (progressMonitor.BeginTask(taskName, 1))
{
if (progressMonitor.IsCanceled)
return;
FileInfo testDriverScriptFile = GetTestDriverScriptFile(testPackage);
if (testDriverScriptFile == null)
return;
HostSetup hostSetup = CreateHostSetup(testPackage);
ScriptRuntimeSetup scriptRuntimeSetup = CreateScriptRuntimeSetup(testPackage);
string testDriverScriptPath = testDriverScriptFile.FullName;
var remoteMessageSink = new RemoteMessageSink(messageSink);
var remoteLogger = new RemoteLogger(logger);
using (var remoteProgressMonitor = new RemoteProgressMonitor(progressMonitor.CreateSubProgressMonitor(1)))
{
testIsolationContext.RunIsolatedTask<ExploreOrRunTask>(hostSetup,
(statusMessage) => progressMonitor.SetStatus(statusMessage),
new object[] { testPackage, scriptRuntimeSetup, testDriverScriptPath, testExplorationOptions, testExecutionOptions,
remoteMessageSink, remoteProgressMonitor, remoteLogger });
}
}
}
/// <inheritdoc />
protected override void ExploreImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage,
TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
{
using (progressMonitor.BeginTask("Exploring unsupported tests.", Math.Max(testPackage.Files.Count, 1)))
{
PublishTestModelFromFiles(testPackage.Files, messageSink, progressMonitor);
}
}
/// <inheritdoc />
protected override void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage,
TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions,
IMessageSink messageSink, IProgressMonitor progressMonitor)
{
double workItems = Math.Max(testPackage.Files.Count, 1);
using (progressMonitor.BeginTask("Running unsupported tests.", workItems * 2))
{
TestModel testModel = PublishTestModelFromFiles(testPackage.Files, messageSink, progressMonitor);
if (testModel != null)
RunTestModel(testModel, messageSink, progressMonitor, workItems);
}
}
/// <inheritdoc />
sealed protected override void ExploreImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
{
ExploreOrRun(testIsolationContext, testPackage, testExplorationOptions, null, messageSink, progressMonitor,
"Exploring tests.");
}
/// <inheritdoc />
protected override sealed void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
{
ExploreOrRun(testIsolationContext, testPackage, testExplorationOptions, testExecutionOptions, messageSink,
progressMonitor, "Running tests.");
}
/// <inheritdoc />
sealed protected override void RunImpl(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions, TestExecutionOptions testExecutionOptions, IMessageSink messageSink, IProgressMonitor progressMonitor)
{
ExploreOrRun(testIsolationContext, testPackage, testExplorationOptions, testExecutionOptions, messageSink,
progressMonitor, "Running tests.");
}
private void ExploreOrRunAssembly(ITestIsolationContext testIsolationContext, TestPackage testPackage, TestExplorationOptions testExplorationOptions,
TestExecutionOptions testExecutionOptions, RemoteMessageSink remoteMessageSink, IProgressMonitor progressMonitor, string taskName,
FileInfo file)
{
using (progressMonitor.BeginTask(taskName, 100))
{
if (progressMonitor.IsCanceled)
return;
string assemblyPath = file.FullName;
progressMonitor.SetStatus("Getting test assembly metadata.");
AssemblyMetadata assemblyMetadata = AssemblyUtils.GetAssemblyMetadata(assemblyPath, AssemblyMetadataFields.RuntimeVersion);
progressMonitor.Worked(2);
if (progressMonitor.IsCanceled)
return;
if (assemblyMetadata != null)
{
Type driverType = GetType();
object[] driverArguments = GetRemoteTestDriverArguments();
HostSetup hostSetup = CreateHostSetup(testPackage, assemblyPath, assemblyMetadata);
using (var remoteProgressMonitor = new RemoteProgressMonitor(
progressMonitor.CreateSubProgressMonitor(97)))
{
testIsolationContext.RunIsolatedTask<ExploreOrRunTask>(hostSetup,
(statusMessage) => progressMonitor.SetStatus(statusMessage),
new object[] { driverType, driverArguments, assemblyPath, testExplorationOptions, testExecutionOptions, remoteMessageSink, remoteProgressMonitor });
}
// Record one final work unit after the isolated task has been fully cleaned up.
progressMonitor.SetStatus("");
progressMonitor.Worked(1);
}
}
}
public IEnumerable <AutoTest.TestRunners.Shared.Results.TestResult> Run(RunSettings settings)
{
if (!_isInitialized)
{
return new AutoTest.TestRunners.Shared.Results.TestResult[] { getNotInitializedResult(settings) }
}
;
var tests = settings.Assembly.Tests.ToList();
var members = settings.Assembly.Members.ToList();
var namespaces = settings.Assembly.Namespaces.ToList();
var runAll = namespaces.Count == 0 && members.Count == 0 && tests.Count == 0;
var steps = new List <TestStepData>();
var testResults = new List <AutoTest.TestRunners.Shared.Results.TestResult>();
// Get a test isolation context. Here we want to run tests in the same AppDomain.
var testIsolationProvider = (ITestIsolationProvider)RuntimeAccessor.ServiceLocator.ResolveByComponentId("Gallio.LocalTestIsolationProvider");
var testIsolationOptions = new TestIsolationOptions();
ITestIsolationContext testIsolationContext = testIsolationProvider.CreateContext(testIsolationOptions, _logger);
var testPackage = new TestPackage();
testPackage.AddFile(new FileInfo(settings.Assembly.Assembly));
testPackage.TestFrameworkFallbackMode = TestFrameworkFallbackMode.Strict;
// Create some test exploration options. Nothing interesting for you here, probably.
var testExplorationOptions = new TestExplorationOptions();
var messageSink = new MessageConsumer()
.Handle <TestStepStartedMessage>((message) =>
{
steps.Add(message.Step);
})
.Handle <TestStepFinishedMessage>(message =>
{
var test = steps.FirstOrDefault(x => x.Id.Equals(message.StepId) && x.IsTestCase);
if (test == null)
{
return;
}
var fixture = string.Format("{0}.{1}", test.CodeReference.NamespaceName, steps.First(x => x.Id.Equals(test.ParentId)).Name);
testResults.Add(new AutoTest.TestRunners.Shared.Results.TestResult(
"MbUnit",
settings.Assembly.Assembly,
fixture,
message.Result.Duration.TotalMilliseconds,
string.Format("{0}.{1}", fixture, test.Name),
convertState(message.Result.Outcome.Status),
message.Result.Outcome.DisplayName));
});
// Provide a progress monitor.
var logProgressMonitorProvider = new LogProgressMonitorProvider(_logger);
var options = new TestExecutionOptions();
options.FilterSet = new Gallio.Model.Filters.FilterSet <ITestDescriptor>(new OrFilter <ITestDescriptor>(getTestFilter(namespaces, members, tests)));
// Run the tests.
logProgressMonitorProvider.Run((progressMonitor) =>
{
_testDriver.Run(testIsolationContext, testPackage, testExplorationOptions, options, messageSink, progressMonitor);
});
return(testResults);
}
