/// <summary>
/// Runs the tests.
/// </summary>
/// <param name="tests">Which tests should be run.</param>
/// <param name="context">Context in which to run tests.</param>
/// <param param name="framework">Where results should be stored.</param>
public void RunTests(IEnumerable<TestCase> tests, IRunContext context, IFrameworkHandle framework)
{
_state = ExecutorState.Running;
foreach (var test in tests)
{
if (_state == ExecutorState.Cancelling)
{
_state = ExecutorState.Cancelled;
return;
}
try
{
var reportDocument = RunOrDebugCatchTest(test.Source, test.FullyQualifiedName, context, framework);
var result = GetTestResultFromReport(test, reportDocument, framework);
framework.RecordResult(result);
}
catch (Exception ex)
{
// Log it and move on. It will show up to the user as a test that hasn't been run.
framework.SendMessage(TestMessageLevel.Error, "Exception occured when processing test case: " + test.FullyQualifiedName);
framework.SendMessage(TestMessageLevel.Informational, "Message: " + ex.Message + "\nStacktrace:" + ex.StackTrace);
}
}
}
// NOTE: an earlier version of this code had a FilterBuilder
// property. This seemed to make sense, because we instantiate
// it in two different places. However, the existence of an
// NUnitTestFilterBuilder, containing a reference to an engine
// service caused our second-level tests of the test executor
// to throw an exception. So if you consider doing this, beware!
#endregion
#region ITestExecutor Implementation
/// <summary>
/// Called by the Visual Studio IDE to run all tests. Also called by TFS Build
/// to run either all or selected tests. In the latter case, a filter is provided
/// as part of the run context.
/// </summary>
/// <param name="sources">Sources to be run.</param>
/// <param name="runContext">Context to use when executing the tests.</param>
/// <param name="frameworkHandle">Test log to send results and messages through</param>
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
#if LAUNCHDEBUGGER
if (!Debugger.IsAttached)
Debugger.Launch();
#endif
Initialize(runContext, frameworkHandle);
try
{
foreach (var source in sources)
{
var assemblyName = source;
if (!Path.IsPathRooted(assemblyName))
assemblyName = Path.Combine(Environment.CurrentDirectory, assemblyName);
TestLog.Info("Running all tests in " + assemblyName);
RunAssembly(assemblyName, TestFilter.Empty);
}
}
catch (Exception ex)
{
if (ex is TargetInvocationException)
ex = ex.InnerException;
TestLog.Error("Exception thrown executing tests", ex);
}
finally
{
TestLog.Info(string.Format("NUnit Adapter {0}: Test execution complete", AdapterVersion));
Unload();
}
}
bool GetTestCaseFilterExpression(IRunContext runContext, LoggerHelper logger, string assemblyFileName, out ITestCaseFilterExpression filter)
{
filter = null;
try
{
// In Microsoft.VisualStudio.TestPlatform.ObjectModel V11 IRunContext provides a TestCaseFilter property
// GetTestCaseFilter only exists in V12+
#if PLATFORM_DOTNET
var getTestCaseFilterMethod = runContext.GetType().GetRuntimeMethod("GetTestCaseFilter", new[] { typeof(IEnumerable<string>), typeof(Func<string, TestProperty>) });
#else
var getTestCaseFilterMethod = runContext.GetType().GetMethod("GetTestCaseFilter");
#endif
if (getTestCaseFilterMethod != null)
filter = (ITestCaseFilterExpression)getTestCaseFilterMethod.Invoke(runContext, new object[] { supportedPropertyNames, null });
return true;
}
catch (TargetInvocationException e)
{
var innerExceptionType = e.InnerException.GetType();
if (innerExceptionType.FullName.EndsWith("TestPlatformFormatException", StringComparison.OrdinalIgnoreCase))
{
logger.LogError("{0}: Exception discovering tests: {1}", Path.GetFileNameWithoutExtension(assemblyFileName), e.InnerException.Message);
return false;
}
throw;
}
}
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
Guard.ArgumentNotNull("sources", sources);
Guard.ArgumentNotNull("runContext", runContext);
Guard.ArgumentNotNull("frameworkHandle", frameworkHandle);
var cleanupList = new List<ExecutorWrapper>();
try
{
RemotingUtility.CleanUpRegisteredChannels();
cancelled = false;
foreach (string source in sources)
if (VsTestRunner.IsXunitTestAssembly(source))
RunTestsInAssembly(cleanupList, source, frameworkHandle);
}
finally
{
Thread.Sleep(1000);
foreach (var executorWrapper in cleanupList)
executorWrapper.Dispose();
}
}
public void RunTests(IEnumerable<string> sources, IRunContext runContext,
IFrameworkHandle frameworkHandle)
{
SetupExecutionPolicy();
IEnumerable<TestCase> tests = PowerShellTestDiscoverer.GetTests(sources, null);
RunTests(tests, runContext, frameworkHandle);
}
public TestCaseFilter(IRunContext runContext, ISet<string> traitNames, TestEnvironment testEnvironment)
{
_runContext = runContext;
_testEnvironment = testEnvironment;
InitProperties(traitNames);
}
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
//Debugger.Launch();
frameworkHandle.SendMessage(TestMessageLevel.Informational, Strings.EXECUTOR_STARTING);
Settings settings = GetSettings(runContext);
foreach (string currentAsssembly in sources.Distinct())
{
try
{
if (!File.Exists(Path.Combine(Path.GetDirectoryName(Path.GetFullPath(currentAsssembly)),"Machine.Specifications.dll")))
{
frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("Machine.Specifications.dll not found for {0}", currentAsssembly));
continue;
}
frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format(Strings.EXECUTOR_EXECUTINGIN, currentAsssembly));
this.executor.RunAssembly(currentAsssembly, settings, uri, frameworkHandle);
}
catch (Exception ex)
{
frameworkHandle.SendMessage(TestMessageLevel.Error, String.Format(Strings.EXECUTOR_ERROR, currentAsssembly, ex.Message));
}
}
frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("Complete on {0} assemblies ", sources.Count()));
}
public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
//Debugger.Launch();
frameworkHandle.SendMessage(TestMessageLevel.Informational, Strings.EXECUTOR_STARTING);
int executedSpecCount = 0;
Settings settings = GetSettings(runContext);
string currentAsssembly = string.Empty;
try
{
foreach (IGrouping<string, TestCase> grouping in tests.GroupBy(x => x.Source)) {
currentAsssembly = grouping.Key;
frameworkHandle.SendMessage(TestMessageLevel.Informational, string.Format(Strings.EXECUTOR_EXECUTINGIN, currentAsssembly));
List<VisualStudioTestIdentifier> testsToRun = grouping.Select(test => test.ToVisualStudioTestIdentifier()).ToList();
this.executor.RunAssemblySpecifications(currentAsssembly, testsToRun, settings, uri, frameworkHandle);
executedSpecCount += grouping.Count();
}
frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format(Strings.EXECUTOR_COMPLETE, executedSpecCount, tests.GroupBy(x => x.Source).Count()));
} catch (Exception ex)
{
frameworkHandle.SendMessage(TestMessageLevel.Error, string.Format(Strings.EXECUTOR_ERROR, currentAsssembly, ex.Message));
}
finally
{
}
}
public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
this.frameworkHandle = frameworkHandle;
var testLogger = new TestLogger(frameworkHandle);
testLogger.SendMainMessage("Execution started");
foreach (var group in tests.GroupBy(t => t.Source))
{
testLogger.SendInformationalMessage(String.Format("Running selected: '{0}'", group.Key));
try
{
using (var sandbox = new Sandbox<Executor>(group.Key))
{
var assemblyDirectory = new DirectoryInfo(Path.GetDirectoryName(group.Key));
Directory.SetCurrentDirectory(assemblyDirectory.FullName);
sandbox.Content.Execute(this, group.Select(t => t.FullyQualifiedName).ToArray());
}
}
catch (Exception ex)
{
testLogger.SendErrorMessage(ex, String.Format("Exception found while executing tests in group '{0}'", group.Key));
// just go on with the next
}
}
testLogger.SendMainMessage("Execution finished");
}
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
this.frameworkHandle = frameworkHandle;
var testLogger = new TestLogger(frameworkHandle);
testLogger.SendMainMessage("Execution started");
foreach (var source in sources)
{
try
{
using (var sandbox = new Sandbox<Executor>(source))
{
testLogger.SendInformationalMessage(String.Format("Running: '{0}'", source));
var assemblyDirectory = new DirectoryInfo(Path.GetDirectoryName(source));
Directory.SetCurrentDirectory(assemblyDirectory.FullName);
sandbox.Content.Execute(this);
}
}
catch (Exception ex)
{
testLogger.SendErrorMessage(ex, String.Format("Exception found while executing tests in source '{0}'", source));
// just go on with the next
}
}
testLogger.SendMainMessage("Execution finished");
}
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
IMessageLogger log = frameworkHandle;
log.Version();
HandlePoorVisualStudioImplementationDetails(runContext, frameworkHandle);
foreach (var assemblyPath in sources)
{
try
{
if (AssemblyDirectoryContainsFixie(assemblyPath))
{
log.Info("Processing " + assemblyPath);
var listener = new VisualStudioListener(frameworkHandle, assemblyPath);
using (var environment = new ExecutionEnvironment(assemblyPath))
{
environment.RunAssembly(new Options(), listener);
}
}
else
{
log.Info("Skipping " + assemblyPath + " because it is not a test assembly.");
}
}
catch (Exception exception)
{
log.Error(exception);
}
}
}
/// <summary>
/// Called by the Visual Studio IDE to run all tests. Also called by TFS Build
/// to run either all or selected tests. In the latter case, a filter is provided
/// as part of the run context.
/// </summary>
/// <param name="sources">Sources to be run.</param>
/// <param name="runContext">Context to use when executing the tests.</param>
/// <param name="frameworkHandle">Test log to send results and messages through</param>
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
testLog.Initialize(frameworkHandle);
Info("executing tests", "started");
try
{
// Ensure any channels registered by other adapters are unregistered
CleanUpRegisteredChannels();
var tfsfilter = new TFSTestFilter(runContext);
testLog.SendDebugMessage("Keepalive:" + runContext.KeepAlive);
if (!tfsfilter.HasTfsFilterValue && runContext.KeepAlive)
frameworkHandle.EnableShutdownAfterTestRun = true;
foreach (var source in sources)
{
using (currentRunner = new AssemblyRunner(testLog, source, tfsfilter))
{
currentRunner.RunAssembly(frameworkHandle);
}
currentRunner = null;
}
}
catch (Exception ex)
{
testLog.SendErrorMessage("Exception " + ex);
}
finally
{
Info("executing tests", "finished");
}
}
/// <summary>
/// Runs the tests.
/// </summary>
/// <param name="tests">Tests to be run.</param>
/// <param name="runContext">Context to use when executing the tests.</param>
/// <param param name="frameworkHandle">Handle to the framework to record results and to do framework operations.</param>
public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
m_cancelled = false;
try
{
foreach (TestCase test in tests)
{
if (m_cancelled)
{
break;
}
frameworkHandle.RecordStart(test);
frameworkHandle.SendMessage(TestMessageLevel.Informational, "Starting external test for " + test.DisplayName);
var testOutcome = RunExternalTest(test, runContext, frameworkHandle, test);
frameworkHandle.RecordResult(testOutcome);
frameworkHandle.SendMessage(TestMessageLevel.Informational, "Test result:" + testOutcome.Outcome.ToString());
}
}
catch(Exception e)
{
frameworkHandle.SendMessage(TestMessageLevel.Error, "Exception during test execution: " +e.Message);
}
}
public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
frameworkHandle.SendMessage(TestMessageLevel.Informational, Strings.EXECUTOR_STARTING);
int executedSpecCount = 0;
string currentAsssembly = string.Empty;
try
{
ISpecificationExecutor specificationExecutor = this.adapterFactory.CreateExecutor();
IEnumerable<IGrouping<string, TestCase>> groupBySource = tests.GroupBy(x => x.Source);
foreach (IGrouping<string, TestCase> grouping in groupBySource)
{
currentAsssembly = grouping.Key;
frameworkHandle.SendMessage(TestMessageLevel.Informational, string.Format(Strings.EXECUTOR_EXECUTINGIN, currentAsssembly));
specificationExecutor.RunAssemblySpecifications(currentAsssembly, MSpecTestAdapter.uri, runContext, frameworkHandle, grouping);
executedSpecCount += grouping.Count();
}
frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format(Strings.EXECUTOR_COMPLETE, executedSpecCount, groupBySource.Count()));
}
catch (Exception ex)
{
frameworkHandle.SendMessage(TestMessageLevel.Error, string.Format(Strings.EXECUTOR_ERROR, currentAsssembly, ex.Message));
}
finally
{
}
}
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
ChutzpahTracer.TraceInformation("Begin Test Adapter Run Tests");
var settingsProvider = runContext.RunSettings.GetSettings(AdapterConstants.SettingsName) as ChutzpahAdapterSettingsProvider;
var settings = settingsProvider != null ? settingsProvider.Settings : new ChutzpahAdapterSettings();
ChutzpahTracingHelper.Toggle(settings.EnabledTracing);
var testOptions = new TestOptions
{
TestLaunchMode =
runContext.IsBeingDebugged ? TestLaunchMode.Custom:
settings.OpenInBrowser ? TestLaunchMode.FullBrowser:
TestLaunchMode.HeadlessBrowser,
CustomTestLauncher = runContext.IsBeingDebugged ? new VsDebuggerTestLauncher() : null,
MaxDegreeOfParallelism = runContext.IsBeingDebugged ? 1 : settings.MaxDegreeOfParallelism,
ChutzpahSettingsFileEnvironments = new ChutzpahSettingsFileEnvironments(settings.ChutzpahSettingsFileEnvironments)
};
testOptions.CoverageOptions.Enabled = runContext.IsDataCollectionEnabled;
var callback = new ParallelRunnerCallbackAdapter(new ExecutionCallback(frameworkHandle, runContext));
testRunner.RunTests(sources, testOptions, callback);
ChutzpahTracer.TraceInformation("End Test Adapter Run Tests");
}
public void Cleanup()
{
_runContext = null;
foreach (var adapter in GetAdapters()) {
adapter.Cleanup();
}
_adapters.Clear();
}
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
List<TestCase> tests = new List<TestCase>();
TestDiscoverer.VisitTests(sources, t => tests.Add(t));
InternalRunTests(tests, runContext, frameworkHandle, null);
}
public ServersTree(IRunContext runContext)
{
Context = runContext;
Children = new ObservableCollection<Node>
{
new LocalHostNode(this),
};
}
public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) {
ValidateArg.NotNull(tests, "tests");
ValidateArg.NotNull(runContext, "runContext");
ValidateArg.NotNull(frameworkHandle, "frameworkHandle");
_cancelRequested.Reset();
RunTestCases(tests, runContext, frameworkHandle);
}
private void RunTests(string source, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
foreach (var result in ExternalTestExecutor.GetTestResults(source, null).Select(c => CreateTestResult(source, c)))
{
frameworkHandle.RecordStart(result.TestCase);
frameworkHandle.RecordResult(result);
frameworkHandle.RecordEnd(result.TestCase, result.Outcome);
}
}
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
RunTests(sources.Select(x => new TestCase(x, ExecutorDetails.Uri, x)
{
CodeFilePath = x,
LineNumber = 2,
DisplayName = "Abra cadabra"
}), runContext, frameworkHandle);
}
public void RunTests(IEnumerable<string> sources, IRunContext runContext, ITestExecutionRecorder testExecutionRecorder)
{
Guard.ArgumentNotNull("sources", sources);
Guard.ArgumentNotNull("runContext", runContext);
Guard.ArgumentNotNull("testExecutionRecorder", testExecutionRecorder);
foreach (string source in sources)
if (VsTestRunner.IsXunitTestAssembly(source))
RunTestsInAssembly(source, runContext, testExecutionRecorder);
}
public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, ITestExecutionRecorder testExecutionRecorder)
{
Guard.ArgumentNotNull("tests", tests);
Guard.ArgumentNotNull("runContext", runContext);
Guard.ArgumentNotNull("testExecutionRecorder", testExecutionRecorder);
foreach (var testCaseGroup in tests.GroupBy(tc => tc.Source))
if (VsTestRunner.IsXunitTestAssembly(testCaseGroup.Key))
RunTestsInAssembly(testCaseGroup.Key, runContext, testExecutionRecorder, testCaseGroup);
}
private void RunTests(IRunContext runContext, IFrameworkHandle testExecutionRecorder)
{
if (runContext.InIsolation)
launcher.TestProject.TestRunnerFactoryName = StandardTestRunnerFactoryNames.IsolatedAppDomain;
var extension = new VSTestWindowExtension(testExecutionRecorder, testCaseFactory, testResultFactory);
launcher.TestProject.AddTestRunnerExtension(extension);
launcher.Run();
}
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
//Debugger.Launch();
// this a temporary hack until I can figure out why running the specs per assembly directly using mspec does not work with a large number of specifications
// when they are run diectly the first 100 or so specs run fine and then an error occurs saying it has taken more than 10 seconds and is being stopped
// for now we just rediscover and run them like that, makes no sense
TestCaseCollector collector = new TestCaseCollector();
this.DiscoverTests(sources, runContext, frameworkHandle, collector);
this.RunTests(collector.TestCases, runContext, frameworkHandle);
}
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
var validSources = from source in sources
where source.EndsWith(StringHelper.GetSearchExpression(), StringComparison.CurrentCultureIgnoreCase)
select source;
foreach (var source in validSources) {
RunTest(frameworkHandle, source);
}
}
public TestGeneratorAdapterTests()
{
testGeneratorDiscoverer = Substitute.For<ITestGeneratorDiscoverer>();
discoveryContext = Substitute.For<IDiscoveryContext>();
discoverySink = Substitute.For<ITestCaseDiscoverySink>();
logger = Substitute.For<IMessageLogger>();
frameworkHandle = Substitute.For<IFrameworkHandle>();
runContext = Substitute.For<IRunContext>();
testGenerator = Substitute.For<ITestGenerator>();
}
public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrameworkHandle testExecutionRecorder)
{
launcher = new TestLauncher();
foreach (var source in sources)
{
launcher.AddFilePattern(source);
}
RunTests(runContext, testExecutionRecorder);
}
private static IEnumerable<KarmaTestResult> GetTestResults(IRunContext runContext)
{
var firstResultFile = Directory.EnumerateFiles(Path.Combine(@"C:\temp", "test-results"), "*.xml").First();
string outputResults = File.ReadAllText(firstResultFile);
var karmaResultsParser = new JUnitResultsParser();
IEnumerable<KarmaTestResult> testResult = karmaResultsParser.Parse(outputResults);
return testResult;
}
void ITestAdapter.Initialize(IRunContext runContext)
{
// We delay inner TAs initialization until Run method because we don't know which test type this is going to be.
m_runContext = runContext;
TestRunConfiguration runConfig = m_runContext.RunConfig.TestRun.RunConfiguration;
Debug.Assert(runConfig != null);
VsIdeHostRunConfigData runConfigHostData = runConfig.HostData[VsIdeHostAdapter.Name] as VsIdeHostRunConfigData;
if (runConfigHostData != null)
{
VsIdeTestHostContext.AdditionalTestData = runConfigHostData.AdditionalTestData;
}
}
private static XDocument RunOrDebugCatchTest(string testBinary, string testSpec, IRunContext runContext, IFrameworkHandle framework)
{
if (runContext.IsBeingDebugged)
{
return(XDocument.Parse(DebugCatchTest(testBinary, testSpec, framework)));
}
else
{
return(XDocument.Parse(RunCatchTests(testBinary, testSpec)));
}
}
public void RunTests(IEnumerable <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
// We'll just punt and run everything in each file that contains the selected tests
var sources = tests.Select(test => test.Source).Distinct();
RunTests(sources, runContext, frameworkHandle);
}
public void Initialize(IRunContext runContext)
{
_runContext = runContext;
_runId = runContext.RunConfig.TestRun.Id;
}
public void RunTests(IEnumerable <string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
RunTestsWithSourcesCallback?.Invoke(sources, runContext, frameworkHandle);
}
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
// HACK: For now, just run them all - we can fix this later :)
RunTests(tests.Select(t => t.Source).Distinct(), runContext, frameworkHandle);
}
void RunTests(IRunContext runContext, IFrameworkHandle frameworkHandle, Stopwatch stopwatch, Func <XunitVisualStudioSettings, IEnumerable <IGrouping <string, TestCase> > > testCaseAccessor)
{
Guard.ArgumentNotNull("runContext", runContext);
Guard.ArgumentNotNull("frameworkHandle", frameworkHandle);
var settings = SettingsProvider.Load();
var shuttingDown = !runContext.KeepAlive || settings.ShutdownAfterRun;
if (runContext.KeepAlive && settings.ShutdownAfterRun)
{
frameworkHandle.EnableShutdownAfterTestRun = true;
}
var toDispose = new List <IDisposable>();
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
lock (stopwatch)
{
frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Execution started", stopwatch.Elapsed));
frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Settings: MaxParallelThreads = {1}, NameDisplay = {2}, ParallelizeAssemblies = {3}, ParallelizeTestCollections = {4}, ShutdownAfterRun = {5}",
stopwatch.Elapsed,
settings.MaxParallelThreads,
settings.NameDisplay,
settings.ParallelizeAssemblies,
settings.ParallelizeTestCollections,
settings.ShutdownAfterRun));
}
}
try
{
RemotingUtility.CleanUpRegisteredChannels();
cancelled = false;
using (AssemblyHelper.SubscribeResolve())
if (settings.ParallelizeAssemblies)
{
testCaseAccessor(settings)
.Select(testCaseGroup => RunTestsInAssemblyAsync(runContext, frameworkHandle, toDispose, testCaseGroup.Key, testCaseGroup, settings, stopwatch))
.ToList()
.ForEach(@event => @event.WaitOne());
}
else
{
testCaseAccessor(settings)
.ToList()
.ForEach(testCaseGroup => RunTestsInAssembly(runContext, frameworkHandle, toDispose, testCaseGroup.Key, testCaseGroup, settings, stopwatch));
}
}
finally
{
if (!shuttingDown)
{
toDispose.ForEach(disposable => disposable.Dispose());
}
}
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
lock (stopwatch)
frameworkHandle.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Execution complete", stopwatch.Elapsed));
}
}
public bool ShouldAttachToTestHost(IEnumerable <VsTestCase> tests, IRunContext runContext)
{
// TODO: expose setting in runContext to attach to testhost if needed?
return(false);
}
private void RunTestCase(
IFrameworkHandle frameworkHandle,
IRunContext runContext,
TestCase test,
Dictionary <string, PythonProjectSettings> sourceToSettings
)
{
var testResult = new TestResult(test);
frameworkHandle.RecordStart(test);
testResult.StartTime = DateTimeOffset.Now;
PythonProjectSettings settings;
if (!sourceToSettings.TryGetValue(test.Source, out settings))
{
sourceToSettings[test.Source] = settings = LoadProjectSettings(test.Source, _interpreterService);
}
if (settings == null)
{
frameworkHandle.SendMessage(
TestMessageLevel.Error,
"Unable to determine interpreter to use for " + test.Source);
RecordEnd(
frameworkHandle,
test,
testResult,
null,
"Unable to determine interpreter to use for " + test.Source,
TestOutcome.Failed);
return;
}
var debugMode = PythonDebugMode.None;
if (runContext.IsBeingDebugged && _app != null)
{
debugMode = settings.EnableNativeCodeDebugging ? PythonDebugMode.PythonAndNative : PythonDebugMode.PythonOnly;
}
var testCase = new PythonTestCase(settings, test, debugMode);
var dte = _app.GetDTE();
if (debugMode != PythonDebugMode.None)
{
dte.Debugger.DetachAll();
}
if (!File.Exists(settings.Factory.Configuration.InterpreterPath))
{
frameworkHandle.SendMessage(TestMessageLevel.Error, "Interpreter path does not exist: " + settings.Factory.Configuration.InterpreterPath);
return;
}
var env = new Dictionary <string, string>();
var pythonPathVar = settings.Factory.Configuration.PathEnvironmentVariable;
var pythonPath = testCase.SearchPaths;
if (!string.IsNullOrWhiteSpace(pythonPathVar))
{
if (_app != null)
{
var settingsManager = SettingsManagerCreator.GetSettingsManager(dte);
if (settingsManager != null)
{
var store = settingsManager.GetReadOnlySettingsStore(SettingsScope.UserSettings);
if (store != null && store.CollectionExists(@"PythonTools\Options\General"))
{
var settingStr = store.GetString(@"PythonTools\Options\General", "ClearGlobalPythonPath", "True");
bool settingBool;
if (bool.TryParse(settingStr, out settingBool) && !settingBool)
{
pythonPath += ";" + Environment.GetEnvironmentVariable(pythonPathVar);
}
}
}
}
env[pythonPathVar] = pythonPath;
}
foreach (var envVar in testCase.Environment)
{
env[envVar.Key] = envVar.Value;
}
using (var proc = ProcessOutput.Run(
!settings.IsWindowsApplication ?
settings.Factory.Configuration.InterpreterPath :
settings.Factory.Configuration.WindowsInterpreterPath,
testCase.Arguments,
testCase.WorkingDirectory,
env,
false,
null
)) {
bool killed = false;
#if DEBUG
frameworkHandle.SendMessage(TestMessageLevel.Informational, "cd " + testCase.WorkingDirectory);
frameworkHandle.SendMessage(TestMessageLevel.Informational, "set " + (pythonPathVar ?? "") + "=" + (pythonPath ?? ""));
frameworkHandle.SendMessage(TestMessageLevel.Informational, proc.Arguments);
#endif
proc.Wait(TimeSpan.FromMilliseconds(500));
if (debugMode != PythonDebugMode.None)
{
if (proc.ExitCode.HasValue)
{
// Process has already exited
frameworkHandle.SendMessage(TestMessageLevel.Error, "Failed to attach debugger because the process has already exited.");
if (proc.StandardErrorLines.Any())
{
frameworkHandle.SendMessage(TestMessageLevel.Error, "Standard error from Python:");
foreach (var line in proc.StandardErrorLines)
{
frameworkHandle.SendMessage(TestMessageLevel.Error, line);
}
}
}
try {
if (debugMode == PythonDebugMode.PythonOnly)
{
string qualifierUri = string.Format("tcp://{0}@localhost:{1}", testCase.DebugSecret, testCase.DebugPort);
while (!_app.AttachToProcess(proc, PythonRemoteDebugPortSupplierUnsecuredId, qualifierUri))
{
if (proc.Wait(TimeSpan.FromMilliseconds(500)))
{
break;
}
}
}
else
{
var engines = new[] { PythonDebugEngineGuid, VSConstants.DebugEnginesGuids.NativeOnly_guid };
while (!_app.AttachToProcess(proc, engines))
{
if (proc.Wait(TimeSpan.FromMilliseconds(500)))
{
break;
}
}
}
#if DEBUG
} catch (COMException ex) {
frameworkHandle.SendMessage(TestMessageLevel.Error, "Error occurred connecting to debuggee.");
frameworkHandle.SendMessage(TestMessageLevel.Error, ex.ToString());
try {
proc.Kill();
} catch (InvalidOperationException) {
// Process has already exited
}
killed = true;
}
#else
} catch (COMException) {
void RunTestsInAssembly(IRunContext runContext,
IFrameworkHandle frameworkHandle,
LoggerHelper logger,
TestPlatformContext testPlatformContext,
RunSettings runSettings,
IMessageSinkWithTypes reporterMessageHandler,
AssemblyRunInfo runInfo)
{
if (cancelled)
{
return;
}
var assemblyDisplayName = "(unknown assembly)";
try
{
var assembly = new XunitProjectAssembly {
AssemblyFilename = runInfo.AssemblyFileName
};
var assemblyFileName = runInfo.AssemblyFileName;
assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
var configuration = runInfo.Configuration;
var shadowCopy = configuration.ShadowCopyOrDefault;
var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
if (runSettings.DisableAppDomain)
{
appDomain = AppDomainSupport.Denied;
}
#if WINDOWS_UAP
// For AppX Apps, use the package location
assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif
var diagnosticSink = DiagnosticMessageSink.ForDiagnostics(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault);
var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink);
using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
{
var testCasesMap = new Dictionary <string, TestCase>();
var testCases = new List <ITestCase>();
if (runInfo.TestCases == null || !runInfo.TestCases.Any())
{
// Discover tests
AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo();
DiscoverTestsInSource(controller, logger, testPlatformContext, runSettings,
(source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled),
(source, discoverer, discoveryOptions, visitor) =>
{
if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault())
{
foreach (var testCase in visitor.TestCases)
{
logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')",
testCase.DisplayName, testCase.UniqueID);
}
}
assemblyDiscoveredInfo = new AssemblyDiscoveredInfo
{
AssemblyFileName = source,
DiscoveredTestCases = GetVsTestCases(source, discoverer, visitor, logger, testPlatformContext)
};
},
assemblyFileName,
shadowCopy,
configuration
);
if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any())
{
if (configuration.InternalDiagnosticMessagesOrDefault)
{
logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName);
}
return;
}
// Filter tests
var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames));
var filter = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames);
var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList();
foreach (var filteredTestCase in filteredTestCases)
{
var uniqueID = filteredTestCase.UniqueID;
if (testCasesMap.ContainsKey(uniqueID))
{
logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName);
}
else
{
testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase);
testCases.Add(filteredTestCase.TestCase);
}
}
}
else
{
try
{
var serializations = runInfo.TestCases
.Select(tc => tc.GetPropertyValue <string>(SerializedTestCaseProperty, null))
.ToList();
if (configuration.InternalDiagnosticMessagesOrDefault)
{
logger.LogWithSource(runInfo.AssemblyFileName, "Deserializing {0} test case(s):{1}{2}",
serializations.Count,
Environment.NewLine,
string.Join(Environment.NewLine, serializations.Select(x => $" {x}")));
}
var deserializedTestCasesByUniqueId = controller.BulkDeserialize(serializations);
if (deserializedTestCasesByUniqueId == null)
{
logger.LogErrorWithSource(assemblyFileName, "Received null response from BulkDeserialize");
}
else
{
for (int idx = 0; idx < runInfo.TestCases.Count; ++idx)
{
try
{
var kvp = deserializedTestCasesByUniqueId[idx];
var vsTestCase = runInfo.TestCases[idx];
if (kvp.Value == null)
{
logger.LogErrorWithSource(assemblyFileName, "Test case {0} failed to deserialize: {1}", vsTestCase.DisplayName, kvp.Key);
}
else
{
testCasesMap.Add(kvp.Key, vsTestCase);
testCases.Add(kvp.Value);
}
}
catch (Exception ex)
{
logger.LogErrorWithSource(assemblyFileName, "Catastrophic error deserializing item #{0}: {1}", idx, ex);
}
}
}
}
catch (Exception ex)
{
logger.LogWarningWithSource(assemblyFileName, "Catastrophic error during deserialization: {0}", ex);
}
}
// Execute tests
var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
if (runSettings.DisableParallelization)
{
executionOptions.SetSynchronousMessageReporting(true);
executionOptions.SetDisableParallelization(true);
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled))
{
IExecutionSink resultsSink = vsExecutionSink;
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink);
}
controller.RunTests(testCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
}
}
}
catch (Exception ex)
{
logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex);
}
}
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
Debugger.Launch();
}
public void RunTests(IEnumerable <string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
Debugger.Launch();
}
/// <summary>
/// Execute the parameter tests present in parameter source
/// </summary>
/// <param name="tests">Tests to execute.</param>
/// <param name="runContext">The run context.</param>
/// <param name="frameworkHandle">Handle to record test start/end/results.</param>
/// <param name="source">The test container for the tests.</param>
/// <param name="isDeploymentDone">Indicates if deployment is done.</param>
private void ExecuteTestsInSource(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle, string source, bool isDeploymentDone)
{
Debug.Assert(!string.IsNullOrEmpty(source), "Source cannot be empty");
if (isDeploymentDone)
{
source = Path.Combine(PlatformServiceProvider.Instance.TestDeployment.GetDeploymentDirectory(), Path.GetFileName(source));
}
using (var isolationHost = PlatformServiceProvider.Instance.CreateTestSourceHost(source, runContext?.RunSettings, frameworkHandle))
{
var testRunner = isolationHost.CreateInstanceForType(
typeof(UnitTestRunner),
new object[] { MSTestSettings.CurrentSettings }) as UnitTestRunner;
PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Created unit-test runner {0}", source);
// Default test set is filtered tests based on user provided filter criteria
IEnumerable <TestCase> testsToRun = Enumerable.Empty <TestCase>();
var filterExpression = this.TestMethodFilter.GetFilterExpression(runContext, frameworkHandle, out var filterHasError);
if (filterHasError)
{
// Bail out without processing everything else below.
return;
}
testsToRun = tests.Where(t => MatchTestFilter(filterExpression, t, this.TestMethodFilter));
// this is done so that appropriate values of testcontext properties are set at source level
// and are merged with session level parameters
var sourceLevelParameters = PlatformServiceProvider.Instance.SettingsProvider.GetProperties(source);
if (this.sessionParameters != null && this.sessionParameters.Count > 0)
{
sourceLevelParameters = sourceLevelParameters.Concat(this.sessionParameters).ToDictionary(x => x.Key, x => x.Value);
}
var sourceSettingsProvider = isolationHost.CreateInstanceForType(
typeof(TestAssemblySettingsProvider),
null) as TestAssemblySettingsProvider;
var sourceSettings = sourceSettingsProvider.GetSettings(source);
var parallelWorkers = sourceSettings.Workers;
var parallelScope = sourceSettings.Scope;
if (MSTestSettings.CurrentSettings.ParallelizationWorkers.HasValue)
{
// The runsettings value takes precedence over an assembly level setting. Reset the level.
parallelWorkers = MSTestSettings.CurrentSettings.ParallelizationWorkers.Value;
}
if (MSTestSettings.CurrentSettings.ParallelizationScope.HasValue)
{
// The runsettings value takes precedence over an assembly level setting. Reset the level.
parallelScope = MSTestSettings.CurrentSettings.ParallelizationScope.Value;
}
if (!MSTestSettings.CurrentSettings.DisableParallelization && sourceSettings.CanParallelizeAssembly && parallelWorkers > 0)
{
// Parallelization is enabled. Let's do further classification for sets.
var logger = (IMessageLogger)frameworkHandle;
logger.SendMessage(
TestMessageLevel.Informational,
string.Format(CultureInfo.CurrentCulture, Resource.TestParallelizationBanner, source, parallelWorkers, parallelScope));
// Create test sets for execution, we can execute them in parallel based on parallel settings
IEnumerable <IGrouping <bool, TestCase> > testsets = Enumerable.Empty <IGrouping <bool, TestCase> >();
// Parallel and not parallel sets.
testsets = testsToRun.GroupBy(t => t.GetPropertyValue <bool>(TestAdapter.Constants.DoNotParallelizeProperty, false));
var parallelizableTestSet = testsets.FirstOrDefault(g => g.Key == false);
var nonparallelizableTestSet = testsets.FirstOrDefault(g => g.Key == true);
if (parallelizableTestSet != null)
{
ConcurrentQueue <IEnumerable <TestCase> > queue = null;
// Chunk the sets into further groups based on parallel level
switch (parallelScope)
{
case ExecutionScope.MethodLevel:
queue = new ConcurrentQueue <IEnumerable <TestCase> >(parallelizableTestSet.Select(t => new[] { t }));
break;
case ExecutionScope.ClassLevel:
queue = new ConcurrentQueue <IEnumerable <TestCase> >(parallelizableTestSet.GroupBy(t => t.GetPropertyValue(TestAdapter.Constants.TestClassNameProperty) as string));
break;
}
var tasks = new List <Task>();
for (int i = 0; i < parallelWorkers; i++)
{
tasks.Add(Task.Factory.StartNew(
() =>
{
while (!queue.IsEmpty)
{
if (this.cancellationToken != null && this.cancellationToken.Canceled)
{
// if a cancellation has been requested, do not queue any more test runs.
break;
}
if (queue.TryDequeue(out IEnumerable <TestCase> testSet))
{
this.ExecuteTestsWithTestRunner(testSet, runContext, frameworkHandle, source, sourceLevelParameters, testRunner);
}
}
},
CancellationToken.None,
TaskCreationOptions.LongRunning,
TaskScheduler.Default));
}
Task.WaitAll(tasks.ToArray());
}
// Queue the non parallel set
if (nonparallelizableTestSet != null)
{
this.ExecuteTestsWithTestRunner(nonparallelizableTestSet, runContext, frameworkHandle, source, sourceLevelParameters, testRunner);
}
}
else
{
this.ExecuteTestsWithTestRunner(testsToRun, runContext, frameworkHandle, source, sourceLevelParameters, testRunner);
}
this.RunCleanup(frameworkHandle, testRunner);
PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo(
"Executed tests belonging to source {0}",
source);
}
}
List <AssemblyRunInfo> GetTests(IEnumerable <string> sources, LoggerHelper logger, IRunContext runContext)
{
// For store apps, the files are copied to the AppX dir, we need to load it from there
#if PLATFORM_DOTNET
sources = sources.Select(s => Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(s)));
#endif
var result = new List <AssemblyRunInfo>();
var knownTraitNames = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
DiscoverTests(
sources,
logger,
(source, discoverer, discoveryOptions) => new VsExecutionDiscoveryVisitor(),
(source, discoverer, discoveryOptions, visitor) =>
{
var vsFilteredTestCases = visitor.TestCases.Select(testCase => VsDiscoveryVisitor.CreateVsTestCase(source, discoverer, testCase, false, logger: logger, knownTraitNames: knownTraitNames)).ToList();
// Apply any filtering
var filterHelper = new TestCaseFilterHelper(knownTraitNames);
vsFilteredTestCases = filterHelper.GetFilteredTestList(vsFilteredTestCases, runContext, logger, source).ToList();
// Re-create testcases with unique names if there is more than 1
var testCases = visitor.TestCases.Where(tc => vsFilteredTestCases.Any(vsTc => vsTc.DisplayName == tc.DisplayName)).GroupBy(tc => $"{tc.TestMethod.TestClass.Class.Name}.{tc.TestMethod.Method.Name}")
.SelectMany(group => group.Select(testCase =>
VsDiscoveryVisitor.CreateVsTestCase(
source,
discoverer,
testCase,
forceUniqueNames: group.Count() > 1,
logger: logger,
knownTraitNames: knownTraitNames))
.Where(vsTestCase => vsTestCase != null)).ToList(); // pre-enumerate these as it populates the known trait names collection
var runInfo = new AssemblyRunInfo
{
AssemblyFileName = source,
Configuration = LoadConfiguration(source),
TestCases = testCases
};
result.Add(runInfo);
}
);
return(result);
}
/// <summary>
/// Gets filter expression from run context.
/// </summary>
/// <param name="context">Run context</param>
/// <returns>Filter expression.</returns>
private ITestCaseFilterExpression GetTestCaseFilterFromRunContext(IRunContext context)
{
return(context.GetTestCaseFilter(this.supportedProperties.Keys, this.PropertyProvider));
}
/// <summary>
/// Implements initialization. This is called by ITestAdapter.Initialize
/// (a synchronous function) which will block until this function is
/// completed.
/// </summary>
/// <param name="runContext">
/// The context for the current test run.
/// </param>
private async Task InitializeWorker(
TestProperties vars,
IRunContext runContext,
ITestElement currentTest = null
)
{
string application, executable, versionString, hive;
Version version;
string launchTimeoutInSecondsString;
int launchTimeoutInSeconds;
string reuseInstanceString;
// VSApplication is the registry key name like 'VisualStudio'
application = vars[VSTestProperties.VSApplication.Key] ?? VSTestProperties.VSApplication.VisualStudio;
// VSExecutableName is the executable name like 'devenv'
if (!vars.TryGetValue(VSTestProperties.VSExecutable.Key, out executable))
{
executable = VSTestProperties.VSExecutable.DevEnv;
}
if (!string.IsNullOrEmpty(executable) &&
string.IsNullOrEmpty(Path.GetExtension(executable)))
{
executable = Path.ChangeExtension(executable, ".exe");
}
// VSVersion is the version like '12.0'
if (!vars.TryGetValue(VSTestProperties.VSVersion.Key, out versionString) ||
!Version.TryParse(versionString, out version))
{
version = new Version(int.Parse(AssemblyVersionInfo.VSVersion), 0);
}
// VSHive is the optional hive like 'Exp'
hive = vars[VSTestProperties.VSHive.Key];
if (!vars.TryGetValue(VSTestProperties.VSLaunchTimeoutInSeconds.Key, out launchTimeoutInSecondsString) ||
!int.TryParse(launchTimeoutInSecondsString, out launchTimeoutInSeconds))
{
launchTimeoutInSeconds = 30;
}
if (string.IsNullOrEmpty(application) || string.IsNullOrEmpty(executable) || version == null)
{
throw new ArgumentException(string.Format(
Resources.MissingConfigurationValues,
application ?? "(null)",
executable ?? "(null)",
version != null ? version.ToString() : "(null)",
hive ?? "(default)"
));
}
_mockVs = (application == VSTestProperties.VSApplication.Mock);
if (_mockVs)
{
_remote = new TesteeTestAdapter();
_remote.Initialize(_runContext);
// In the mock case tester and testee are the same process, therefore
// VSTestContext is in our process too. So we can just set this value
// directly here.
VSTestContext.IsMock = true;
return;
}
if (!vars.TryGetValue(VSTestProperties.VSReuseInstance.Key, out reuseInstanceString) ||
!bool.TryParse(reuseInstanceString, out _reuseInstance))
{
// the default behavior is to reuse the current instance when possible
_reuseInstance = true;
}
// TODO: Detect and perform first run of VS if necessary.
// The first time a VS hive is run, the user sees a dialog allowing
// them to select settings such as the theme. We can avoid this by
// running devenv.exe /resetSettings <path to profile.settings>
// first, though it is not trivial to detect when this is necessary.
// Without having done this, all tests will time out. For now, the
// user is responsible for running VS at least once before
// attempting to execute tests.
var cts = new CancellationTokenSource(TimeSpan.FromSeconds(launchTimeoutInSeconds));
try {
await Connect(application, executable, version, hive, runContext, currentTest, cts.Token);
} catch (OperationCanceledException ex) {
throw new TimeoutException(string.Format(Resources.VSLaunchTimeout, launchTimeoutInSeconds), ex);
} catch (Exception ex) {
throw new InvalidOperationException(
string.Format(Resources.VSFailedToLaunch, application, executable, version, hive),
ex
);
}
}
public void RunTests(IEnumerable <string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
}
public FeatureRunner(IStringStepRunner stringStepRunner, IRunContext context)
{
this.context = context;
this.stringStepRunner = stringStepRunner;
}
private void RunTestsCore(IEnumerable <ITestScript> tests, Predicate <ITestFunction> filter, ILogger logger, IRunContext runContext, ITestExecutionRecorder recorder)
{
var settingsService = runContext.RunSettings.GetSettings(PSycheTestRunSettings.SettingsProviderName) as IPSycheTestSettingsService ?? new PSycheTestSettingsService();
var testCaseMap = new Dictionary <ITestFunction, TestCase>();
var executor = _executorFactory(logger);
executor.OutputDirectory = new DirectoryInfo(runContext.TestRunDirectory);
logger.Info("Test output directory: {0}", executor.OutputDirectory.FullName);
foreach (var module in settingsService.Settings.Modules)
{
logger.Info("Adding module '{0}'", module);
executor.InitialModules.Add(module);
}
executor.TestStarting += (o, e) =>
{
var testCase = _mapper.Map(e.Test);
testCaseMap[e.Test] = testCase;
recorder.RecordStart(testCase);
};
executor.TestEnded += (o, e) =>
{
var testCase = testCaseMap[e.Test];
recorder.RecordEnd(testCase, _mapper.Map(e.Result.Status));
recorder.RecordResult(_mapper.Map(testCase, e.Result));
};
using (_cancellationTokenSource = new CancellationTokenSource())
{
executor.ExecuteAsync(tests, filter, _cancellationTokenSource.Token).Wait(); // Awaiting seems to cause odd behavior, just block instead.
}
_cancellationTokenSource = null;
}
void RunTestsInAssembly(IRunContext runContext,
IFrameworkHandle frameworkHandle,
LoggerHelper logger,
TestPlatformContext testPlatformContext,
IMessageSinkWithTypes reporterMessageHandler,
AssemblyRunInfo runInfo)
{
if (cancelled)
{
return;
}
var assembly = new XunitProjectAssembly {
AssemblyFilename = runInfo.AssemblyFileName
};
var assemblyFileName = runInfo.AssemblyFileName;
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
var configuration = runInfo.Configuration;
var shadowCopy = configuration.ShadowCopyOrDefault;
var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
if (RunSettingsHelper.DisableAppDomain)
{
appDomain = AppDomainSupport.Denied;
}
try
{
#if WINDOWS_UAP
// For AppX Apps, use the package location
assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif
var diagnosticSink = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault);
var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink);
using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
{
var testCasesMap = new Dictionary <string, TestCase>();
var testCases = new List <ITestCase>();
if (runInfo.TestCases == null || !runInfo.TestCases.Any())
{
// Discover tests
AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo();
DiscoverTestsInSource(controller, logger, testPlatformContext,
(source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled),
(source, discoverer, discoveryOptions, visitor) =>
{
if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault())
{
foreach (var testCase in visitor.TestCases)
{
logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')",
testCase.DisplayName, testCase.UniqueID);
}
}
assemblyDiscoveredInfo = new AssemblyDiscoveredInfo
{
AssemblyFileName = source,
DiscoveredTestCases = visitor.TestCases.Select(testCase => new DiscoveredTestCase(source, discoverer, testCase, logger, testPlatformContext)).ToList()
};
},
assemblyFileName,
shadowCopy,
configuration
);
if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any())
{
if (configuration.InternalDiagnosticMessagesOrDefault)
{
logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName);
}
return;
}
// Filter tests
var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames));
// Apply any filtering
var filter = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames);
var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList();
// Force unique names if there is more than 1 testcase with the same name
foreach (var groupWithDuplicateNames in filteredTestCases.GroupBy(dtc => dtc.Name).Where(group => group.Count() > 1))
{
foreach (var discoveredTestCaseWithDuplicateName in groupWithDuplicateNames)
{
discoveredTestCaseWithDuplicateName.ForceUniqueName();
}
}
foreach (var filteredTestCase in filteredTestCases)
{
var uniqueID = filteredTestCase.UniqueID;
if (testCasesMap.ContainsKey(uniqueID))
{
logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName);
}
else
{
testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase);
testCases.Add(filteredTestCase.TestCase);
}
}
}
else
{
// We are in Run Specific tests scenario, the `TestCase` objects are available.
// Query the `TestCase` objects to find XunitTestCase objects.
foreach (var vstestCase in runInfo.TestCases)
{
var xunitTestCase = Deserialize(logger, controller, vstestCase);
if (xunitTestCase != null)
{
testCasesMap.Add(xunitTestCase.UniqueID, vstestCase);
testCases.Add(xunitTestCase);
}
}
}
// Execute tests
var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
if (RunSettingsHelper.DisableParallelization)
{
executionOptions.SetSynchronousMessageReporting(true);
executionOptions.SetDisableParallelization(true);
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled))
{
IExecutionSink resultsSink = vsExecutionSink;
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink);
}
controller.RunTests(testCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
}
}
}
catch (Exception ex)
{
logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex);
}
}
private FeatureResults Run(IFeatureRunner featureRunner, IEnumerable <Feature> features, IRunContext context)
{
var result = new FeatureResults();
foreach (Feature feature in features)
{
FeatureResult featureResult = null;
try
{
context.FeatureStarted(feature);
featureResult = featureRunner.Run(feature);
result.Add(featureResult);
}
finally
{
context.FeatureFinished(featureResult);
}
}
return(result);
}
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
RunTestsWithTestsCallback?.Invoke(tests, runContext, frameworkHandle);
}
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
_mCancelled = false;
SetupExecutionPolicy();
var testSets = new List <TestCaseSet>();
foreach (var testCase in tests)
{
var describe = testCase.FullyQualifiedName.Split('.').First();
var codeFile = testCase.CodeFilePath;
var testSet = testSets.FirstOrDefault(m => m.Describe.Equals(describe, StringComparison.OrdinalIgnoreCase) &&
m.File.Equals(codeFile, StringComparison.OrdinalIgnoreCase));
if (testSet == null)
{
testSet = new TestCaseSet(codeFile, describe);
testSets.Add(testSet);
}
testSet.TestCases.Add(testCase);
}
foreach (var testSet in testSets)
{
if (_mCancelled)
{
break;
}
var testOutput = new StringBuilder();
try
{
var testAdapter = new TestAdapterHost();
testAdapter.HostUi.OutputString = s =>
{
if (!string.IsNullOrEmpty(s))
{
testOutput.Append(s);
}
};
var runpsace = RunspaceFactory.CreateRunspace(testAdapter);
runpsace.Open();
using (var ps = PowerShell.Create())
{
ps.Runspace = runpsace;
RunTestSet(ps, testSet, runContext);
foreach (var testResult in testSet.TestResults)
{
frameworkHandle.RecordResult(testResult);
}
}
}
catch (Exception ex)
{
foreach (var testCase in testSet.TestCases)
{
var testResult = new TestResult(testCase);
testResult.Outcome = TestOutcome.Failed;
testResult.ErrorMessage = ex.Message;
testResult.ErrorStackTrace = ex.StackTrace;
frameworkHandle.RecordResult(testResult);
}
}
if (testOutput.Length > 0)
{
frameworkHandle.SendMessage(TestMessageLevel.Informational, testOutput.ToString());
}
}
}
/// <summary>
/// ITestAdapter method: called just before the test run finishes and
/// gives the adapter a chance to do any clean-up.
/// </summary>
/// <param name="runContext">The run context for this run</param>
void ITestAdapter.PreTestRunFinished(IRunContext runContext)
{
HostSide.PreTestRunFinished(runContext);
}
public void PreTestRunFinished(IRunContext runContext)
{
RemoteCall(r => r.PreTestRunFinished(runContext));
}
public void RunTests(IEnumerable <string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
throw new NotImplementedException();
}
private async Task Connect(
string application,
string executable,
Version version,
string hive,
IRunContext runContext,
ITestElement currentTest,
CancellationToken cancel
)
{
var hiveOption = string.IsNullOrEmpty(hive) ? "" : (" /rootSuffix " + hive);
if (_reuseInstance &&
_ide != null &&
_remote != null &&
application == _currentApplication &&
executable == _currentExecutable &&
version == _currentVersion &&
hive == _currentHive)
{
if (runContext != null)
{
SendMessage(
runContext,
string.Format(Resources.VSReuseMessage, application, executable, version, hiveOption),
currentTest
);
}
return;
}
Close();
if (runContext != null)
{
SendMessage(
runContext,
string.Format(Resources.VSLaunchMessage, application, executable, version, hiveOption),
currentTest
);
}
Internal.VisualStudio ide = null;
try {
ide = await Internal.VisualStudio.LaunchAsync(application, executable, version, hive, cancel);
var p = Process.GetProcessById(ide.ProcessId);
var url = string.Format("ipc://{0}/{1}", VSTestHostPackage.GetChannelName(p), TesteeTestAdapter.Url);
try {
_remote = (TesteeTestAdapter)RemotingServices.Connect(typeof(TesteeTestAdapter), url);
} catch (RemotingException ex) {
throw new InvalidOperationException(Resources.FailedToConnect, ex);
}
_currentApplication = application;
_currentExecutable = executable;
_currentVersion = version;
_currentHive = hive;
_ide = ide;
ide = null;
} finally {
if (ide != null)
{
ide.Dispose();
}
}
}
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
throw new NotImplementedException();
}
private string RunProtractor(TestCase test, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
var resultFile = Path.GetFileNameWithoutExtension(test.CodeFilePath);
resultFile += ".result.json";
resultFile = AppConfig.ResultsPath ?? Path.GetTempPath() + Path.DirectorySeparatorChar + resultFile;
frameworkHandle.SendMessage(TestMessageLevel.Informational, "Framework: Using result file: " + resultFile);
// Can't use test.Source as it's going to be in lowercase
var cwd = Helper.FindInDirectoryTree(test.CodeFilePath, "package.json");
var exe = Helper.FindExePath(AppConfig.Program);
//frameworkHandle.SendMessage(TestMessageLevel.Error, "Framework: Exe " + exe);
//frameworkHandle.SendMessage(TestMessageLevel.Error, "Framework: Cwd " + cwd);
//frameworkHandle.SendMessage(TestMessageLevel.Error, "Framework: ResultFile " + resultFile);
ProcessStartInfo info = new ProcessStartInfo()
{
Arguments = string.Format("{0} --resultJsonOutputFile \"{1}\" --specs \"{2}\"", AppConfig.Arguments, resultFile, test.CodeFilePath),
FileName = exe,
WorkingDirectory = cwd,//runContext.SolutionDirectory,
UseShellExecute = false,
CreateNoWindow = true,
RedirectStandardError = true,
RedirectStandardOutput = true,
};
frameworkHandle.SendMessage(TestMessageLevel.Informational, $"Framework: Starting {exe} on '{cwd}' with arguments: {info.Arguments}");
Process p = new Process
{
StartInfo = info
};
using (AutoResetEvent outputWaitHandle = new AutoResetEvent(false))
using (AutoResetEvent errorWaitHandle = new AutoResetEvent(false))
{
p.OutputDataReceived += (sender, e) => {
if (e.Data == null)
{
outputWaitHandle.Set();
}
else if (!String.IsNullOrWhiteSpace(e.Data))
{
// Console.WriteLine(e.Data);
frameworkHandle.SendMessage(TestMessageLevel.Informational, e.Data);
}
};
p.ErrorDataReceived += (sender, e) =>
{
if (e.Data == null)
{
errorWaitHandle.Set();
}
else if (!string.IsNullOrWhiteSpace(e.Data))
{
// Console.Error.WriteLine(e.Data);
frameworkHandle.SendMessage(TestMessageLevel.Error, e.Data);
}
};
p.Start();
p.BeginOutputReadLine();
p.BeginErrorReadLine();
p.WaitForExit();
outputWaitHandle.WaitOne(10000);
errorWaitHandle.WaitOne(10000);
// Process completed. Check process.ExitCode here.
frameworkHandle.SendMessage(TestMessageLevel.Informational, "Framework: Complete. Exit code: " + p.ExitCode.ToString());
}
return(resultFile);
}
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
}
public void RunTests(IEnumerable <TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
var explicitlyInclude = CommonStringPrefix.Of(tests.Select(t => t.FullyQualifiedName));
InternalRunTests(tests, runContext, frameworkHandle, explicitlyInclude);
}
