public void DiscoverTests(IEnumerable<string> sources,
IDiscoveryContext discoveryContext,
IMessageLogger logger,
ITestCaseDiscoverySink discoverySink)
{
var validSources = from source in sources
where source.EndsWith(StringHelper.GetSearchExpression(), StringComparison.CurrentCultureIgnoreCase)
select source;
foreach (var source in validSources) {
var results = appDomainRunner.ExecuteSpecifications(source);
var query = from result in results
from @group in result.Examples
from example in @group.Examples
select example;
foreach (var example in query) {
var testCase = new TestCase(example.Reason, DefaultTestExecutor.ExecutorUri, source) {
CodeFilePath = example.FileName,
LineNumber = example.LineNumber
};
discoverySink.SendTestCase(testCase);
}
}
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext
, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
#if DEBUG && LAUNCH_DEBUGGER
System.Diagnostics.Debugger.Launch();
#endif
try
{
Logger.Initialize(logger);
var sourceGroups = sources.GroupBy(Path.GetExtension);
foreach (IGrouping<string, string> sourceGroup in sourceGroups)
{
IIutestTestDiscoverer discoverer = iutestTestDiscovererFactory.GetTestDiscoverer(sourceGroup.Key);
if( discoverer != null )
{
discoverer.DiscoverTests(sourceGroup, discoveryContext, logger, discoverySink);
}
}
}
catch( Exception e )
{
Logger.Error("iutest_adapter: Exception caught while DiscoverTests: {0}, {1} ", e.Message, e.HResult );
Logger.Error(e.StackTrace);
}
finally
{
Logger.Uninitialize();
}
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
ChutzpahTracer.TraceInformation("Begin Test Adapter Discover Tests");
var settingsProvider = discoveryContext.RunSettings.GetSettings(AdapterConstants.SettingsName) as ChutzpahAdapterSettingsProvider;
var settings = settingsProvider != null ? settingsProvider.Settings : new ChutzpahAdapterSettings();
ChutzpahTracingHelper.Toggle(settings.EnabledTracing);
var testOptions = new TestOptions
{
MaxDegreeOfParallelism = settings.MaxDegreeOfParallelism,
ChutzpahSettingsFileEnvironments = new ChutzpahSettingsFileEnvironments(settings.ChutzpahSettingsFileEnvironments)
};
IList<TestError> errors;
var testCases = testRunner.DiscoverTests(sources, testOptions, out errors);
ChutzpahTracer.TraceInformation("Sending discovered tests to test case discovery sink");
foreach (var testCase in testCases)
{
var vsTestCase = testCase.ToVsTestCase();
discoverySink.SendTestCase(vsTestCase);
}
foreach (var error in errors)
{
logger.SendMessage(TestMessageLevel.Error, RunnerCallback.FormatFileErrorMessage(error));
}
ChutzpahTracer.TraceInformation("End Test Adapter Discover Tests");
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, ITestCaseDiscoverySink discoverySink)
{
Code.Require(sources, "sources");
Code.Require(discoverySink, "discoverySink");
if (this.Settings.DiscoveryMethodType == DiscoveryMethodType.DiscoveryListContent)
{
// Delegate to ListContentDiscoverer
ListContentDiscoverer discoverer = new ListContentDiscoverer(new ExternalBoostTestRunnerFactory(), VSProvider);
discoverer.DiscoverTests(sources, discoveryContext, discoverySink);
}
else
{
foreach (string source in sources)
{
TestFramework framework = DiscoverTestFramework(source);
if ((framework != null) && (framework.MasterTestSuite != null))
{
VSDiscoveryVisitor visitor = new VSDiscoveryVisitor(source, discoverySink);
framework.MasterTestSuite.Apply(visitor);
}
}
}
}
public void DiscoverTests(IEnumerable<string> sources,
IDiscoveryContext discoveryContext,
IMessageLogger logger,
ITestCaseDiscoverySink discoverySink)
{
GetTests(sources, discoverySink);
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
testLogger = new TestLogger(logger);
testLogger.SendMainMessage("Discovery started");
foreach (var source in sources)
{
testLogger.SendDebugMessage(String.Format("Processing: '{0}'", source));
try
{
using (var sandbox = new Sandbox<Discoverer>(source))
{
if (sandbox.Content != null)
{
sandbox.Content
.DiscoverTests()
.Select(name => name.ToTestCase(source))
.ForEach(discoverySink.SendTestCase);
}
}
}
catch (Exception ex)
{
testLogger.SendErrorMessage(ex, String.Format("Exception found while discovering tests in source '{0}'", source));
// just go on with the next
}
}
testLogger.SendMainMessage("Discovery finished");
}
public void DiscoverTests(IEnumerable<string> sources
, IDiscoveryContext discoveryContext
, Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging.IMessageLogger logger
, ITestCaseDiscoverySink discoverySink)
{
GetTests(sources, discoverySink);
}
public void Load(IDiscoveryContext context)
{
if (context == null)
throw new ArgumentNullException("context", "Load called with null context");
Load(context.RunSettings.SettingsXml);
}
public virtual void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
TestAdapterInfo = CreateTestAdapterInfo();
var testLogger = TestAdapterInfo.CreateLogger(logger);
var discoverLogger = new TestLogger(testLogger, "Discover");
var testSettings = discoveryContext.RunSettings.GetTestSettings(TestAdapterInfo.SettingsName);
var count = 0;
var testCount = 0;
foreach (var source in sources)
{
var sourceSettings = GetSourceSettings(source, testSettings);
if (sourceSettings != null)
{
testCount += DiscoverTests(sourceSettings, testLogger, discoverySink).Result;
count += 1;
}
else
{
discoverLogger.Warn("Could not get settings for {0}", source);
}
}
if (count > 0 || testCount > 0)
{
discoverLogger.Info("{0} tests discovered in {1} test containers", testCount, count);
}
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger log, ITestCaseDiscoverySink discoverySink)
{
log.Version();
RemotingUtility.CleanUpRegisteredChannels();
foreach (var assemblyPath in sources)
{
try
{
if (AssemblyDirectoryContainsFixie(assemblyPath))
{
log.Info("Processing " + assemblyPath);
using (var environment = new ExecutionEnvironment(assemblyPath))
{
var methodGroups = environment.DiscoverTestMethodGroups(new Options());
foreach (var methodGroup in methodGroups)
discoverySink.SendTestCase(new TestCase(methodGroup.FullName, VsTestExecutor.Uri, assemblyPath));
}
}
else
{
log.Info("Skipping " + assemblyPath + " because it is not a test assembly.");
}
}
catch (Exception exception)
{
log.Error(exception);
}
}
}
/// <summary>
/// Finds tests in Catch unit test binaries. Note: We have to run the binary to enumerate tests.
/// </summary>
/// <param name="sources">Binaries to search for tests.</param>
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext context, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
currentLogger = logger;
foreach (var source in sources)
{
switch (checkCatchTestBinaryFormat(source))
{
case CatchBinaryFormat.NoCatchTestBinary:
continue;
case CatchBinaryFormat.CatchTestBinary:
foreach (var testCase in ListTestsInBinary(source))
{
discoverySink.SendTestCase(testCase);
}
continue;
case CatchBinaryFormat.CatchTestBinaryWithXmlTestList:
foreach (var testCase in ListTestsInBinaryXml(source))
{
discoverySink.SendTestCase(testCase);
}
break;
}
}
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
Code.Require(sources, "sources");
Code.Require(discoverySink, "discoverySink");
BoostTestAdapterSettings settings = BoostTestAdapterSettingsProvider.GetSettings(discoveryContext);
_sourceFilters = SourceFilterFactory.Get(settings);
IDictionary<string, ProjectInfo> solutioninfo = null;
var numberOfAttempts = 100;
// try several times to overcome "Application is Busy" COMException
while (numberOfAttempts > 0)
{
try
{
solutioninfo = PrepareTestCaseData(sources);
// set numberOfAttempts = 0, because there is no need to try again,
// since obviously no exception was thrown at this point
numberOfAttempts = 0;
}
catch (COMException)
{
--numberOfAttempts;
// re-throw after all attempts have failed
if (numberOfAttempts == 0)
{
throw;
}
}
}
GetBoostTests(solutioninfo, discoverySink);
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
_logger = logger;
_logger.SendMessage(TestMessageLevel.Informational, ">>> DiscoverTests");
GetTests(sources, discoverySink);
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) {
ValidateArg.NotNull(sources, "sources");
ValidateArg.NotNull(discoverySink, "discoverySink");
var settings = discoveryContext.RunSettings;
DiscoverTests(sources, logger, discoverySink, settings);
}
/// <summary>
/// @todo add more metadata to test cases (however ctest alone does not provide everything needed)
/// </summary>
/// <param name="sources"></param>
/// <param name="discoveryContext"></param>
/// <param name="logger"></param>
/// <param name="discoverySink"></param>
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging.IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
foreach (var source in sources)
{
var testcase = CTestCase.Parse(source);
if (testcase == null) continue;
discoverySink.SendTestCase(testcase);
}
}
public TestGeneratorAdapterTests()
{
testGeneratorDiscoverer = Substitute.For<ITestGeneratorDiscoverer>();
discoveryContext = Substitute.For<IDiscoveryContext>();
discoverySink = Substitute.For<ITestCaseDiscoverySink>();
logger = Substitute.For<IMessageLogger>();
frameworkHandle = Substitute.For<IFrameworkHandle>();
runContext = Substitute.For<IRunContext>();
testGenerator = Substitute.For<ITestGenerator>();
}
/// <summary>
/// Applies the discovery procedure over the provided sources
/// </summary>
/// <param name="provider">An IVisualStudioInstanceProvider instance</param>
/// <param name="sources">The sources which to discover tests from</param>
/// <param name="context">The IDiscoveryContext to use</param>
/// <returns>An enumeration of discovered test cases</returns>
private IEnumerable<VSTestCase> Discover(IVisualStudioInstanceProvider provider, IEnumerable<string> sources, IDiscoveryContext context)
{
ConsoleMessageLogger logger = new ConsoleMessageLogger();
DefaultTestCaseDiscoverySink sink = new DefaultTestCaseDiscoverySink();
IBoostTestDiscoverer discoverer = new SourceCodeDiscoverer(provider);
discoverer.DiscoverTests(sources, context, logger, sink);
return sink.Tests;
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
foreach(var source in sources) {
var xDomainSink = new TestAdapterLogger(logger, source);
xDomainSink.OnBeginTest += (_, e) => discoverySink.SendTestCase(e.TestCase);
CrossDomainConeRunner.WithProxyInDomain<ConeTestAdapterProxy, int>(string.Empty,
new [] { source, },
proxy => proxy.DiscoverTests(source, xDomainSink)
);
}
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger, ITestCaseDiscoverySink discoverySink)
{
testLog.Initialize(messageLogger);
Info("discovering tests", "started");
// Ensure any channels registered by other adapters are unregistered
CleanUpRegisteredChannels();
foreach (string sourceAssembly in sources)
{
testLog.SendDebugMessage("Processing " + sourceAssembly);
TestRunner runner = new TestDomain();
TestPackage package = new TestPackage(sourceAssembly);
TestConverter testConverter = null;
try
{
if (runner.Load(package))
{
testConverter = new TestConverter(testLog, sourceAssembly);
int cases = ProcessTestCases(runner.Test, discoverySink, testConverter);
testLog.SendDebugMessage(string.Format("Discovered {0} test cases", cases));
}
else
{
testLog.NUnitLoadError(sourceAssembly);
}
}
catch (System.BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
testLog.AssemblyNotSupportedWarning(sourceAssembly);
}
catch (System.IO.FileNotFoundException ex)
{
// Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
testLog.DependentAssemblyNotFoundWarning(ex.FileName, sourceAssembly);
}
catch (System.Exception ex)
{
testLog.SendErrorMessage("Exception thrown discovering tests in " + sourceAssembly, ex);
}
finally
{
if (testConverter != null)
testConverter.Dispose();
testConverter = null;
runner.Unload();
}
}
Info("discovering test", "finished");
}
/// <summary>
/// ITestDiscover, Given a list of test sources this method pulls out the test cases
/// </summary>
/// <param name="sources">List of test sources passed from client (Client can be VS or command line)</param>
/// <param name="discoveryContext">Context and runSettings for current run. Discoverer pulls out the tests based on current context</param>
/// <param name="logger">Used to relay messages to registered loggers</param>
/// <param name="discoverySink">Callback used to notify client upon discovery of test cases</param>
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) {
ValidateArg.NotNull(sources, "sources");
ValidateArg.NotNull(discoverySink, "discoverySink");
ValidateArg.NotNull(logger, "logger");
using (var buildEngine = new MSBuild.ProjectCollection()) {
try {
// Load all the test containers passed in (.njsproj msbuild files)
foreach (string source in sources) {
buildEngine.LoadProject(source);
}
foreach (var proj in buildEngine.LoadedProjects) {
var projectHome = Path.GetFullPath(Path.Combine(proj.DirectoryPath, "."));
Dictionary<string, List<TestFileEntry>> testItems = new Dictionary<string,List<TestFileEntry>>(StringComparer.OrdinalIgnoreCase);
// Provide all files to the test analyzer
foreach (var item in proj.Items.Where(item => item.ItemType == "Compile" || item.ItemType == "TypeScriptCompile")) {
//Check to see if this is a TestCase
string value = item.GetMetadataValue("TestFramework");
if (!TestContainerDiscoverer.IsValidTestFramework(value)) {
continue;
}
string fileAbsolutePath = CommonUtils.GetAbsoluteFilePath(projectHome, item.EvaluatedInclude);
bool typeScriptTest = TypeScript.TypeScriptHelpers.IsTypeScriptFile(fileAbsolutePath);
if(typeScriptTest){
fileAbsolutePath = TypeScript.TypeScriptHelpers.GetTypeScriptBackedJavaScriptFile(proj,fileAbsolutePath);
} else if (!Path.GetExtension(fileAbsolutePath).Equals(".js", StringComparison.OrdinalIgnoreCase)) {
continue;
}
List<TestFileEntry> fileList;
if (!testItems.TryGetValue(value, out fileList)){
fileList = new List<TestFileEntry>();
testItems.Add(value, fileList);
}
fileList.Add(new TestFileEntry(fileAbsolutePath,typeScriptTest));
}
//Debug.Fail("Before Discover");
DiscoverTests(testItems, proj, discoverySink, logger);
}
} catch (Exception ex) {
logger.SendMessage(TestMessageLevel.Error, ex.Message);
throw;
} finally {
// Disposing buildEngine does not clear the document cache in
// VS 2013, so manually unload all projects before disposing.
buildEngine.UnloadAllProjects();
}
}
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, ITestCaseDiscoverySink discoverySink)
{
Code.Require(sources, "sources");
Code.Require(discoverySink, "discoverySink");
if (this.Settings.DiscoveryMethodType == DiscoveryMethodType.DiscoveryListContent)
{
// Delegate to ListContentDiscoverer
ListContentDiscoverer discoverer = new ListContentDiscoverer(new ExternalBoostTestRunnerFactory(), VSProvider);
discoverer.DiscoverTests(sources, discoveryContext, discoverySink);
}
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
Guard.ArgumentNotNull("sources", sources);
Guard.ArgumentNotNull("logger", logger);
Guard.ArgumentNotNull("discoverySink", discoverySink);
try
{
RemotingUtility.CleanUpRegisteredChannels();
List<SourceSink> sourceSinks = new List<SourceSink>();
using (AssemblyHelper.SubscribeResolve())
try
{
foreach (string source in sources)
{
try
{
if (cancelled)
break;
if (IsXunitTestAssembly(source))
{
var framework = new Xunit2(SourceInformationProvider, source, configFileName: null, shadowCopy: true);
var sink = new VsDiscoveryVisitor(source, framework, logger, discoverySink, () => cancelled);
sourceSinks.Add(new SourceSink { Framework = framework, Source = source, Sink = sink });
framework.Find(includeSourceInformation: true, messageSink: sink);
}
}
catch (Exception e)
{
logger.SendMessage(TestMessageLevel.Error, String.Format("xUnit.net: Exception discovering tests from {0}: {1}", source, e));
}
}
foreach (var sourceSink in sourceSinks)
sourceSink.Sink.Finished.WaitOne();
}
finally
{
foreach (var sourceSink in sourceSinks)
{
sourceSink.Sink.Dispose();
sourceSink.Framework.Dispose();
}
}
}
catch
{
}
}
/// <summary>
/// Builds a BoostTestAdapterSettings structure based on the information located within the IDiscoveryContext instance.
/// </summary>
/// <param name="context">The discovery context instance</param>
/// <returns>A BoostTestRunnerSettings instance based on the information identified via the provided IDiscoveryContext instance.</returns>
public static BoostTestAdapterSettings GetSettings(IDiscoveryContext context)
{
Utility.Code.Require(context, "context");
BoostTestAdapterSettings settings = new BoostTestAdapterSettings();
BoostTestAdapterSettingsProvider provider = (context.RunSettings == null) ? null : context.RunSettings.GetSettings(BoostTestAdapterSettings.XmlRootName) as BoostTestAdapterSettingsProvider;
if (provider != null)
{
settings = provider.Settings;
}
// Return defaults
return settings;
}
/// <summary>
/// Method called by Visual Studio (discovered via reflection) for test enumeration
/// </summary>
/// <param name="sources">path, target name and target extensions to discover</param>
/// <param name="discoveryContext">discovery context settings</param>
/// <param name="logger"></param>
/// <param name="discoverySink">Unit test framework Sink</param>
/// <remarks>Entry point of the discovery procedure</remarks>
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
#if DEBUG && LAUNCH_DEBUGGER
System.Diagnostics.Debugger.Launch();
#endif
if (sources == null)
return;
Logger.Initialize(logger);
DiscoverTests(sources, discoveryContext, discoverySink);
Logger.Shutdown();
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
try
{
foreach (var source in sources)
{
IEnumerable<TestCase> tests = GetTests(source);
tests.ToList().ForEach(x => discoverySink.SendTestCase(x));
}
}
catch (Exception ex)
{
Trace.WriteLine("an error occured: " + ex.ToString());
}
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
foreach (var source in sources)
{
using (var sandbox = new Sandbox<Discoverer>(source))
{
if (sandbox.Content != null)
{
sandbox.Content
.DiscoverTests()
.Select(name => name.ToTestCase(source))
.ForEach(discoverySink.SendTestCase);
}
}
}
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
Utility.Code.Require(sources, "sources");
Utility.Code.Require(discoverySink, "discoverySink");
foreach (string source in sources)
{
TestFramework framework = DiscoverTestFramework(source);
if ((framework != null) && (framework.MasterTestSuite != null))
{
BoostTestCaseDiscoverer frameworkDiscoverer = new BoostTestCaseDiscoverer(source, discoverySink);
framework.MasterTestSuite.Apply(frameworkDiscoverer);
}
}
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger,
ITestCaseDiscoverySink discoverySink)
{
var settingsProvider =
discoveryContext.RunSettings.GetSettings(VSTestSettings.SettingsName) as VSTestSettingsService;
var settings = settingsProvider != null && settingsProvider.Settings != null
? settingsProvider.Settings
: new VSTestSettings() {WorkingDirectory = @"C:\Source"};
VSLogger logger = new VSLogger(messageLogger);
foreach (var source in sources)
{
logger.Information(string.Format("Beginning test discovery on {0}", source));
GTestExtractor extractor = new GTestExtractor(false);
ITestSuite suite = extractor.ExtractFrom(source);
foreach (var testCase in suite.TestCases)
{
foreach (var test in testCase.Tests)
{
logger.Information(string.Format("Found test {0}", test.FullyQualifiedName));
discoverySink.SendTestCase(test.ToVSTest());
System.Threading.Thread.Sleep(0);
}
}
//ProcessStartInfo listTestsInfo = new ProcessStartInfo();
//listTestsInfo.FileName = source;
//listTestsInfo.Arguments = "--gtest_list_tests";
//listTestsInfo.UseShellExecute = false;
//listTestsInfo.CreateNoWindow = true;
//listTestsInfo.RedirectStandardOutput = true;
//Process listTestsProc = new Process();
//listTestsProc.StartInfo = listTestsInfo;
//logger.SendMessage(TestMessageLevel.Informational, string.Format("Beginning test discovery on {0}", source));
//listTestsProc.Start();
//listTestsProc.WaitForExit(5000);
//ParseTests(listTestsProc.StandardOutput, discoverySink, logger, source);
}
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
var settingsProvider = discoveryContext.RunSettings.GetSettings(ChutzpahAdapterSettings.SettingsName) as ChutzpahAdapterSettingsService;
var settings = settingsProvider != null ? settingsProvider.Settings : new ChutzpahAdapterSettings();
var testOptions = new TestOptions
{
TestFileTimeoutMilliseconds = settings.TimeoutMilliseconds,
TestingMode = settings.TestingMode,
MaxDegreeOfParallelism = settings.MaxDegreeOfParallelism
};
var testCases = testRunner.DiscoverTests(sources, testOptions);
foreach (var testCase in testCases)
{
var vsTestCase = testCase.ToVsTestCase();
discoverySink.SendTestCase(vsTestCase);
}
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger,
ITestCaseDiscoverySink discoverySink)
{
logger.SendMessage(TestMessageLevel.Informational, Process.GetCurrentProcess().ProcessName + " Id: " + Process.GetCurrentProcess().Id.ToString());
foreach (var source in sources)
{
logger.SendMessage(TestMessageLevel.Informational, source);
}
try
{
GetTests(sources, discoverySink);
}
catch (Exception e)
{
logger.SendMessage(TestMessageLevel.Error, "exception thrown during test discovery: " + e.Message);
}
}
public void DiscoverTests(IEnumerable <string> executables, IDiscoveryContext discoveryContext,
IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
Stopwatch stopwatch = Stopwatch.StartNew();
if (_settings == null || _settings.GetType() == typeof(SettingsWrapper)) // check whether we have a mock
{
CommonFunctions.CreateEnvironment(discoveryContext.RunSettings,
logger, out _logger, out _settings);
}
_discoverer = new GoogleTestDiscoverer(_logger, _settings);
if (!IsSupportedVisualStudioVersion())
{
return;
}
CommonFunctions.LogVisualStudioVersion(_logger);
_logger.LogInfo(Common.Resources.TestDiscoveryStarting);
_logger.DebugInfo(String.Format(Resources.Settings, _settings));
if (_settings.SkipOriginCheck)
{
_logger.LogWarning($"Option '{SettingsWrapper.OptionSkipOriginCheck}' is true - this might impose a security risk to your system");
}
try
{
var reporter = new VsTestFrameworkReporter(discoverySink, _logger);
_discoverer.DiscoverTests(executables, reporter);
stopwatch.Stop();
_logger.LogInfo(String.Format(Resources.TestDiscoveryCompleted, stopwatch.Elapsed));
}
catch (Exception e)
{
_logger.LogError(String.Format(Resources.TestDiscoveryExceptionError, e));
}
CommonFunctions.ReportErrors(_logger, TestPhase.TestDiscovery, _settings.OutputMode, _settings.SummaryMode);
}
public void Discover(IDiscoveryContext context)
{
if (IsNullable)
{
if (Presence == Presence.Forbidden)
{
context.AddError(ForbiddenErrorId, true);
return;
}
if (Presence == Presence.Required)
{
context.AddError(RequiredErrorId, true);
}
}
for (var i = 0; i < CommandScopes.Count; ++i)
{
CommandScopes[i].Discover(context);
}
}
// The Adapter is constructed using the default constructor.
// We don't have any info to initialize it until one of the
// ITestDiscovery or ITestExecutor methods is called. Each
// Discover or Execute method must call this method.
protected void Initialize(IDiscoveryContext context, IMessageLogger messageLogger)
{
var engine = new TestEngineClass();
InternalEngineCreated?.Invoke(engine);
TestEngine = engine;
TestLog = new TestLogger(messageLogger);
Settings = new AdapterSettings(TestLog);
TestLog.InitSettings(Settings);
try
{
Settings.Load(context);
TestLog.Verbosity = Settings.Verbosity;
InitializeForbiddenFolders();
SetCurrentWorkingDirectory();
}
catch (Exception e)
{
TestLog.Warning("Error initializing RunSettings. Default settings will be used");
TestLog.Warning(e.ToString());
}
}
/// <summary>
/// Gets filter expression from discovery context.
/// </summary>
/// <param name="context">Discovery context</param>
/// <param name="logger">The logger to log exception messages too.</param>
/// <returns>Filter expression.</returns>
private ITestCaseFilterExpression GetTestCaseFilterFromDiscoveryContext(IDiscoveryContext context, IMessageLogger logger)
{
try
{
// GetTestCaseFilter is present in DiscoveryContext but not in IDiscoveryContext interface.
MethodInfo methodGetTestCaseFilter = context.GetType().GetRuntimeMethod("GetTestCaseFilter", new[] { typeof(IEnumerable <string>), typeof(Func <string, TestProperty>) });
return((ITestCaseFilterExpression)methodGetTestCaseFilter?.Invoke(context, new object[] { this.supportedProperties.Keys, (Func <string, TestProperty>) this.PropertyProvider }));
}
catch (Exception ex)
{
// In case of UWP .Net Native Tool Chain compilation. Invoking methods via Reflection doesn't work, hence discovery always fails.
// Hence throwing exception only if it is of type TargetInvocationException(i.e. Method got invoked but something went wrong in GetTestCaseFilter Method)
if (ex is TargetInvocationException)
{
throw ex.InnerException;
}
logger.SendMessage(TestMessageLevel.Warning, ex.Message);
}
return(null);
}
void ITestDiscoverer.DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
Guard.ArgumentNotNull("sources", sources);
Guard.ArgumentNotNull("logger", logger);
Guard.ArgumentNotNull("discoverySink", discoverySink);
Guard.ArgumentValid("sources", "AppX/MSIX not supported for discovery", !ContainsAppX(sources));
var stopwatch = Stopwatch.StartNew();
var loggerHelper = new LoggerHelper(logger, stopwatch);
PrintHeader(loggerHelper);
var runSettings = RunSettings.Parse(discoveryContext?.RunSettings?.SettingsXml);
if (!runSettings.IsMatchingTargetFramework())
{
return;
}
var testPlatformContext = new TestPlatformContext
{
// Discovery from command line (non designmode) never requires source information
// since there is no session or command line runner doesn't send back VSTestCase objects
// back to adapter.
RequireSourceInformation = runSettings.CollectSourceInformation,
// Command line runner could request for Discovery in case of running specific tests. We need
// the XunitTestCase serialized in this scenario.
RequireXunitTestProperty = true
};
var testCaseFilter = new TestCaseFilter(discoveryContext, loggerHelper);
DiscoverTests(
sources, loggerHelper, testPlatformContext, runSettings,
(source, discoverer, discoveryOptions) => new VsDiscoverySink(source, discoverer, loggerHelper, discoverySink, discoveryOptions, testPlatformContext, testCaseFilter, () => cancelled)
);
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, ITestCaseDiscoverySink discoverySink)
{
if (sources == null)
{
return;
}
BoostTestAdapterSettings settings = BoostTestAdapterSettingsProvider.GetSettings(discoveryContext);
try
{
// Filter out any sources which are not of interest
if (!TestSourceFilter.IsNullOrEmpty(settings.Filters))
{
sources = sources.Where(settings.Filters.ShouldInclude);
}
var results = _boostTestDiscovererFactory.GetDiscoverers(sources.ToList(), settings);
if (results == null)
{
return;
}
// Test discovery
foreach (var discoverer in results)
{
if (discoverer.Sources.Count > 0)
{
Logger.Info("Discovering ({0}): -> [{1}]", discoverer.Discoverer.GetType().Name, string.Join(", ", discoverer.Sources));
discoverer.Discoverer.DiscoverTests(discoverer.Sources, discoveryContext, discoverySink);
}
}
}
catch (Exception ex)
{
Logger.Exception(ex, "Exception caught while discovering tests: {0} ({1})", ex.Message, ex.HResult);
}
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, ITestCaseDiscoverySink discoverySink)
{
Code.Require(sources, "sources");
Code.Require(discoverySink, "discoverySink");
BoostTestAdapterSettings settings = BoostTestAdapterSettingsProvider.GetSettings(discoveryContext);
_sourceFilters = SourceFilterFactory.Get(settings);
IDictionary <string, ProjectInfo> solutioninfo = null;
var numberOfAttempts = 100;
// try several times to overcome "Application is Busy" COMException
while (numberOfAttempts > 0)
{
try
{
solutioninfo = PrepareTestCaseData(sources);
// set numberOfAttempts = 0, because there is no need to try again,
// since obviously no exception was thrown at this point
numberOfAttempts = 0;
}
catch (COMException)
{
--numberOfAttempts;
// re-throw after all attempts have failed
if (numberOfAttempts == 0)
{
throw;
}
}
}
GetBoostTests(solutioninfo, discoverySink);
}
ManualResetEvent RunTestsInAssemblyAsync(IDiscoveryContext discoveryContext,
IFrameworkHandle frameworkHandle,
List <IDisposable> toDispose,
string assemblyFileName,
IEnumerable <TestCase> testCases,
XunitVisualStudioSettings settings,
Stopwatch stopwatch)
{
var @event = new ManualResetEvent(initialState: false);
ThreadPool.QueueUserWorkItem(_ =>
{
try
{
RunTestsInAssembly(discoveryContext, frameworkHandle, toDispose, assemblyFileName, testCases, settings, stopwatch);
}
finally
{
@event.Set();
}
});
return(@event);
}
public void DiscoverTests(IEnumerable <string> executables, IDiscoveryContext discoveryContext,
IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
Stopwatch stopwatch = Stopwatch.StartNew();
if (_settings == null || _settings.GetType() == typeof(SettingsWrapper)) // check whether we have a mock
{
CommonFunctions.CreateEnvironment(discoveryContext.RunSettings,
logger, out _logger, out _settings);
_discoverer = new GoogleTestDiscoverer(_logger, _settings);
}
if (!IsSupportedVisualStudioVersion())
{
return;
}
CommonFunctions.LogVisualStudioVersion(_logger);
_logger.LogInfo("Google Test Adapter: Test discovery starting...");
_logger.DebugInfo($"Solution settings: {_settings}");
try
{
var reporter = new VsTestFrameworkReporter(discoverySink, _logger);
_discoverer.DiscoverTests(executables, reporter);
stopwatch.Stop();
_logger.LogInfo($"Test discovery completed, overall duration: {stopwatch.Elapsed}");
}
catch (Exception e)
{
_logger.LogError($"Exception while discovering tests: {e}");
}
CommonFunctions.ReportErrors(_logger, "test discovery", _settings.DebugMode);
}
public void DiscoverTests(IEnumerable <string> executables, IDiscoveryContext discoveryContext,
IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
ILogger loggerAdapter = new VsTestFrameworkLogger(logger);
if (TestEnvironment == null || TestEnvironment.Options.GetType() == typeof(Options)) // check whether we have a mock
{
var settingsProvider = discoveryContext.RunSettings.GetSettings(GoogleTestConstants.SettingsName) as RunSettingsProvider;
RunSettings ourRunSettings = settingsProvider != null ? settingsProvider.Settings : new RunSettings();
TestEnvironment = new TestEnvironment(new Options(ourRunSettings, loggerAdapter), loggerAdapter);
Discoverer = new GoogleTestDiscoverer(TestEnvironment);
}
try
{
VsTestFrameworkReporter reporter = new VsTestFrameworkReporter(discoverySink);
Discoverer.DiscoverTests(executables, reporter);
}
catch (Exception e)
{
TestEnvironment.LogError("Exception while discovering tests: " + e);
}
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
GetTests(sources, discoverySink);
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger, ITestCaseDiscoverySink discoverySink)
{
#if LAUNCHDEBUGGER
if (!Debugger.IsAttached)
{
Debugger.Launch();
}
#endif
Initialize(discoveryContext, messageLogger);
TestLog.Info(string.Format("NUnit Adapter {0}: Test discovery starting", AdapterVersion));
// Ensure any channels registered by other adapters are unregistered
CleanUpRegisteredChannels();
if (Settings.InProcDataCollectorsAvailable && sources.Count() > 1)
{
TestLog.Error("Unexpected to discover tests in multiple assemblies when InProcDataCollectors specified in run configuration.");
Unload();
return;
}
foreach (string sourceAssembly in sources)
{
var sourceAssemblyPath = Path.IsPathRooted(sourceAssembly) ? sourceAssembly : Path.Combine(Directory.GetCurrentDirectory(), sourceAssembly);
TestLog.Debug("Processing " + sourceAssembly);
ITestRunner runner = null;
try
{
runner = GetRunnerFor(sourceAssemblyPath);
XmlNode topNode = runner.Explore(TestFilter.Empty);
// Currently, this will always be the case but it might change
if (topNode.Name == "test-run")
{
topNode = topNode.FirstChild;
}
if (topNode.GetAttribute("runstate") == "Runnable")
{
var testConverter = new TestConverter(TestLog, sourceAssemblyPath, Settings.CollectSourceInformation);
int cases = ProcessTestCases(topNode, discoverySink, testConverter);
TestLog.Debug(string.Format("Discovered {0} test cases", cases));
// Only save if seed is not specified in runsettings
// This allows workaround in case there is no valid
// location in which the seed may be saved.
if (cases > 0 && !Settings.RandomSeedSpecified)
{
Settings.SaveRandomSeed(Path.GetDirectoryName(sourceAssemblyPath));
}
}
else
{
var msgNode = topNode.SelectSingleNode("properties/property[@name='_SKIPREASON']");
if (msgNode != null && (new[] { "contains no tests", "Has no TestFixtures" }).Any(msgNode.GetAttribute("value").Contains))
{
TestLog.Info("Assembly contains no NUnit 3.0 tests: " + sourceAssembly);
}
else
{
TestLog.Info("NUnit failed to load " + sourceAssembly);
}
}
}
catch (BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.Warning("Assembly not supported: " + sourceAssembly);
}
catch (FileNotFoundException ex)
{
// Either the NUnit framework was not referenced by the test assembly
// or some other error occured. Not a problem if not an NUnit assembly.
TestLog.Warning("Dependent Assembly " + ex.FileName + " of " + sourceAssembly + " not found. Can be ignored if not a NUnit project.");
}
catch (FileLoadException ex)
{
// Attempts to load an invalid assembly, or an assembly with missing dependencies
TestLog.Warning("Assembly " + ex.FileName + " loaded through " + sourceAssembly + " failed. Assembly is ignored. Correct deployment of dependencies if this is an error.");
}
catch (TypeLoadException ex)
{
if (ex.TypeName == "NUnit.Framework.Api.FrameworkController")
{
TestLog.Warning(" Skipping NUnit 2.x test assembly");
}
else
{
TestLog.Warning("Exception thrown discovering tests in " + sourceAssembly, ex);
}
}
catch (Exception ex)
{
TestLog.Warning("Exception thrown discovering tests in " + sourceAssembly, ex);
}
finally
{
if (runner != null)
{
if (runner.IsTestRunning)
{
runner.StopRun(true);
}
runner.Unload();
runner.Dispose();
}
}
}
TestLog.Info(string.Format("NUnit Adapter {0}: Test discovery complete", AdapterVersion));
Unload();
}
public void DiscoverTests(IEnumerable <string> sources,
IDiscoveryContext discoveryContext,
IMessageLogger logger,
ITestCaseDiscoverySink discoverySink)
{
this._log = logger;
this.Log(TestMessageLevel.Informational, "discovering ...");
var v = sources as IList <string> ?? sources.ToList();
// verify we have a CMakeCache.txt directory
var cacheDir = TestContainerHelper.FindCMakeCacheDirectory(v.First());
if (!cacheDir.Any())
{
this.Log(TestMessageLevel.Informational, "cmake cache not found");
return;
}
// read parameters
var cfg = CTestAdapterConfig.ReadFromDisk(Path.Combine(cacheDir, Constants.CTestAdapterConfigFileName)) ??
CTestAdapterConfig.ReadFromCache(cacheDir);
if (null == cfg)
{
this.Log(TestMessageLevel.Error, "could not create CTestAdapterConfig");
return;
}
// make sure a configuration is set
if (!cfg.ActiveConfiguration.Any())
{
if (cfg.TrySetActiveConfigFromConfigTypes())
{
this.Log(TestMessageLevel.Warning,
"Configuration fallback to: " + cfg.ActiveConfiguration);
}
else
{
this.Log(TestMessageLevel.Error, "could not set Configuration");
return;
}
}
this.Log(TestMessageLevel.Informational, "using configuration: " + cfg.ActiveConfiguration);
// make sure we have a ctest executable
if (!File.Exists(cfg.CTestExecutable))
{
cfg.CTestExecutable = TestContainerHelper.FindCTestExe(cfg.CacheDir);
}
if (!File.Exists(cfg.CTestExecutable))
{
this.Log(TestMessageLevel.Error,
"ctest not found, tried: \"" + cfg.CTestExecutable + "\"");
return;
}
this.Log(TestMessageLevel.Informational, "using ctest binary: " + cfg.CTestExecutable);
// collect all existing tests by executing ctest
var collection = TestContainerHelper.FindAllTestsWithCtest(cfg);
foreach (var source in v)
{
var cases = TestContainerHelper.ParseTestContainerFile(source, this._log, collection, cfg.ActiveConfiguration);
foreach (var c in cases)
{
discoverySink.SendTestCase(c.Value);
}
}
this.Log(TestMessageLevel.Informational, "discovering done");
}
/// <summary>
/// Discovers the tests from the source.
/// </summary>
/// <param name="sources">Source files to get the tests from.</param>
/// <returns>Tests from the file.</returns>
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
GetTests(sources, discoverySink);
Console.WriteLine("From within the discoverer : {0}", Environment.CurrentDirectory);
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
AssemblyResolver.SetupHandler();
this.DiscoverTestsCore(sources, logger, discoverySink);
}
public VsDiscoveryVisitor(string source, ITestFrameworkDiscoverer discoverer, IMessageLogger logger, IDiscoveryContext discoveryContext, ITestCaseDiscoverySink discoverySink, Func <bool> cancelThunk)
{
this.source = source;
this.discoverer = discoverer;
this.logger = logger;
this.discoverySink = discoverySink;
this.cancelThunk = cancelThunk;
settings = SettingsProvider.Load();
var settingsProvider = discoveryContext.RunSettings.GetSettings(XunitTestRunSettings.SettingsName) as XunitTestRunSettingsProvider;
if (settingsProvider != null && settingsProvider.Settings != null)
{
settings.Merge(settingsProvider.Settings);
}
}
internal void SendTestCases(string source, IEnumerable <UnitTestElement> testElements, ITestCaseDiscoverySink discoverySink, IDiscoveryContext discoveryContext, IMessageLogger logger)
{
var shouldCollectSourceInformation = MSTestSettings.RunConfigurationSettings.CollectSourceInformation;
var navigationSessions = new Dictionary <string, object>();
try
{
if (shouldCollectSourceInformation)
{
navigationSessions.Add(source, PlatformServiceProvider.Instance.FileOperations.CreateNavigationSession(source));
}
// Get filter expression and skip discovery in case filter expression has parsing error.
ITestCaseFilterExpression filterExpression = this.TestMethodFilter.GetFilterExpression(discoveryContext, logger, out var filterHasError);
if (filterHasError)
{
return;
}
foreach (var testElement in testElements)
{
var testCase = testElement.ToTestCase();
// Filter tests based on test case filters
if (filterExpression != null && filterExpression.MatchTestCase(testCase, (p) => this.TestMethodFilter.PropertyValueProvider(testCase, p)) == false)
{
continue;
}
if (shouldCollectSourceInformation)
{
string testSource = testElement.TestMethod.DeclaringAssemblyName ?? source;
if (!navigationSessions.TryGetValue(testSource, out var testNavigationSession))
{
testNavigationSession = PlatformServiceProvider.Instance.FileOperations.CreateNavigationSession(testSource);
navigationSessions.Add(testSource, testNavigationSession);
}
if (testNavigationSession != null)
{
var className = testElement.TestMethod.DeclaringClassFullName
?? testElement.TestMethod.FullClassName;
var methodName = testElement.TestMethod.Name;
// If it is async test method use compiler generated type and method name for navigation data.
if (!string.IsNullOrEmpty(testElement.AsyncTypeName))
{
className = testElement.AsyncTypeName;
// compiler generated method name is "MoveNext".
methodName = "MoveNext";
}
PlatformServiceProvider.Instance.FileOperations.GetNavigationData(
testNavigationSession,
className,
methodName,
out var minLineNumber,
out var fileName);
if (!string.IsNullOrEmpty(fileName))
{
testCase.LineNumber = minLineNumber;
testCase.CodeFilePath = fileName;
}
}
}
discoverySink.SendTestCase(testCase);
}
}
finally
{
foreach (object navigationSession in navigationSessions.Values)
{
PlatformServiceProvider.Instance.FileOperations.DisposeNavigationSession(navigationSession);
}
}
}
protected override void RunDiscovery(IDiscoveryContext context)
{
context.EnterScope <TMember>(ScopeId);
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
IsDiscoverTestCalled = true;
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger, ITestCaseDiscoverySink discoverySink)
{
Initialize(discoveryContext, messageLogger);
CheckIfDebug();
TestLog.Info($"NUnit Adapter {AdapterVersion}: Test discovery starting");
// Ensure any channels registered by other adapters are unregistered
CleanUpRegisteredChannels();
if (Settings.InProcDataCollectorsAvailable && sources.Count() > 1)
{
TestLog.Error("Unexpected to discover tests in multiple assemblies when InProcDataCollectors specified in run configuration.");
Unload();
return;
}
foreach (string sourceAssembly in sources)
{
string sourceAssemblyPath = Path.IsPathRooted(sourceAssembly) ? sourceAssembly : Path.Combine(Directory.GetCurrentDirectory(), sourceAssembly);
TestLog.Debug("Processing " + sourceAssembly);
if (Settings.DumpXmlTestDiscovery)
{
dumpXml = new DumpXml(sourceAssemblyPath);
}
try
{
var package = CreateTestPackage(sourceAssemblyPath, null);
NUnitEngineAdapter.CreateRunner(package);
var results = NUnitEngineAdapter.Explore();
dumpXml?.AddString(results.AsString());
if (results.IsRunnable)
{
int cases;
using (var testConverter = new TestConverterForXml(TestLog, sourceAssemblyPath, Settings))
{
var timing = new TimingLogger(Settings, TestLog);
cases = ProcessTestCases(results, discoverySink, testConverter);
timing.LogTime("Discovery/Processing/Converting:");
}
TestLog.Debug($"Discovered {cases} test cases");
// Only save if seed is not specified in runsettings
// This allows workaround in case there is no valid
// location in which the seed may be saved.
if (cases > 0 && !Settings.RandomSeedSpecified)
{
Settings.SaveRandomSeed(Path.GetDirectoryName(sourceAssemblyPath));
}
}
else
{
if (results.HasNoNUnitTests)
{
if (Settings.Verbosity > 0)
{
TestLog.Info("Assembly contains no NUnit 3.0 tests: " + sourceAssembly);
}
}
else
{
TestLog.Info("NUnit failed to load " + sourceAssembly);
}
}
}
catch (NUnitEngineException e)
{
if (e.InnerException is BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.Warning("Assembly not supported: " + sourceAssembly);
}
else
{
TestLog.Warning("Exception thrown discovering tests in " + sourceAssembly, e);
}
}
catch (BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.Warning("Assembly not supported: " + sourceAssembly);
}
catch (FileNotFoundException ex)
{
// Either the NUnit framework was not referenced by the test assembly
// or some other error occurred. Not a problem if not an NUnit assembly.
TestLog.Warning("Dependent Assembly " + ex.FileName + " of " + sourceAssembly + " not found. Can be ignored if not an NUnit project.");
}
catch (FileLoadException ex)
{
// Attempts to load an invalid assembly, or an assembly with missing dependencies
TestLog.Warning("Assembly " + ex.FileName + " loaded through " + sourceAssembly + " failed. Assembly is ignored. Correct deployment of dependencies if this is an error.");
}
catch (TypeLoadException ex)
{
if (ex.TypeName == "NUnit.Framework.Api.FrameworkController")
{
TestLog.Warning(" Skipping NUnit 2.x test assembly");
}
else
{
TestLog.Warning("Exception thrown discovering tests in " + sourceAssembly, ex);
}
}
catch (Exception ex)
{
TestLog.Warning("Exception thrown discovering tests in " + sourceAssembly, ex);
}
finally
{
dumpXml?.DumpForDiscovery();
NUnitEngineAdapter?.CloseRunner();
}
}
TestLog.Info($"NUnit Adapter {AdapterVersion}: Test discovery complete");
Unload();
}
/// <summary>
/// ITestDiscover, Given a list of test sources this method pulls out the test cases
/// </summary>
/// <param name="sources">List of test sources passed from client (Client can be VS or command line)</param>
/// <param name="discoveryContext">Context and runSettings for current run. Discoverer pulls out the tests based on current context</param>
/// <param name="logger">Used to relay messages to registered loggers</param>
/// <param name="discoverySink">Callback used to notify client upon discovery of test cases</param>
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
ValidateArg.NotNull(sources, "sources");
ValidateArg.NotNull(discoverySink, "discoverySink");
ValidateArg.NotNull(logger, "logger");
var env = new Dictionary <string, string>();
var root = Environment.GetEnvironmentVariable(NodejsConstants.NodeToolsVsInstallRootEnvironmentVariable);
if (!string.IsNullOrEmpty(root))
{
env["VsInstallRoot"] = root;
env["MSBuildExtensionsPath32"] = Path.Combine(root, "MSBuild");
}
using (var buildEngine = new MSBuild.ProjectCollection(env))
{
try
{
// Load all the test containers passed in (.njsproj msbuild files)
foreach (var source in sources)
{
buildEngine.LoadProject(source);
}
foreach (var proj in buildEngine.LoadedProjects)
{
var projectHome = Path.GetFullPath(Path.Combine(proj.DirectoryPath, "."));
var testItems = new Dictionary <string, List <TestFileEntry> >(StringComparer.OrdinalIgnoreCase);
// Provide all files to the test analyzer
foreach (var item in proj.Items.Where(item => item.ItemType == "Compile" || item.ItemType == "TypeScriptCompile"))
{
//Check to see if this is a TestCase
var value = item.GetMetadataValue("TestFramework");
if (!TestContainerDiscoverer.IsValidTestFramework(value))
{
continue;
}
var fileAbsolutePath = CommonUtils.GetAbsoluteFilePath(projectHome, item.EvaluatedInclude);
var typeScriptTest = TypeScript.TypeScriptHelpers.IsTypeScriptFile(fileAbsolutePath);
if (typeScriptTest)
{
fileAbsolutePath = TypeScript.TypeScriptHelpers.GetTypeScriptBackedJavaScriptFile(proj, fileAbsolutePath);
}
else if (!Path.GetExtension(fileAbsolutePath).Equals(".js", StringComparison.OrdinalIgnoreCase))
{
continue;
}
List <TestFileEntry> fileList;
if (!testItems.TryGetValue(value, out fileList))
{
fileList = new List <TestFileEntry>();
testItems.Add(value, fileList);
}
fileList.Add(new TestFileEntry(fileAbsolutePath, typeScriptTest));
}
DiscoverTests(testItems, proj, discoverySink, logger);
}
}
catch (Exception ex)
{
logger.SendMessage(TestMessageLevel.Error, ex.Message);
throw;
}
finally
{
// Disposing buildEngine does not clear the document cache in
// VS 2013, so manually unload all projects before disposing.
buildEngine.UnloadAllProjects();
}
}
}
public TestDiscoveryTests()
{
_context = Substitute.For <IDiscoveryContext>();
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
// we can ignore the sources argument since this will be a collection of assemblies.
ValidateArg.NotNull(discoverySink, nameof(discoverySink));
ValidateArg.NotNull(logger, nameof(logger));
// extract the project file from the discovery context.
var unitTestSettings = new UnitTestSettings(discoveryContext.RunSettings);
if (string.IsNullOrEmpty(unitTestSettings.TestSource))
{
// no need to log since the test executor will report 'no tests'
return;
}
if (string.IsNullOrEmpty(unitTestSettings.TestFrameworksLocation))
{
logger.SendMessage(TestMessageLevel.Error, "Failed to locate the test frameworks.");
return;
}
sources = new[] { unitTestSettings.TestSource };
var frameworkDiscoverer = new FrameworkDiscoverer(unitTestSettings.TestFrameworksLocation);
var projects = new List <(string projectFilePath, IEnumerable <XElement> propertyGroup)>();
// There's an issue when loading the project using the .NET Core msbuild bits,
// so we load the xml, and extract the properties we care about.
// Downside is we only have the raw contents of the XmlElements, i.e. we don't
// expand any variables.
// The issue we encountered is that the msbuild implementation was not able to
// locate the SDK targets/props files. See: https://github.com/Microsoft/msbuild/issues/3434
try
{
foreach (var source in sources)
{
var cleanPath = source.Trim('\'', '"');
// we only support project files, e.g. csproj, vbproj, etc.
if (!cleanPath.EndsWith("proj", StringComparison.OrdinalIgnoreCase))
{
continue;
}
var project = XDocument.Load(cleanPath);
// structure looks like Project/PropertyGroup/JsTestRoot and Project/PropertyGroup/JsTestFramework
var propertyGroup = project.Descendants("Project").Descendants("PropertyGroup");
projects.Add((cleanPath, propertyGroup));
}
foreach (var(projectFile, propertyGroup) in projects)
{
var testFramework = propertyGroup.Descendants(NodeProjectProperty.TestFramework).FirstOrDefault()?.Value;
var testRoot = propertyGroup.Descendants(NodeProjectProperty.TestRoot).FirstOrDefault()?.Value;
var outDir = propertyGroup.Descendants(NodeProjectProperty.TypeScriptOutDir).FirstOrDefault()?.Value ?? "";
if (string.IsNullOrEmpty(testRoot) || string.IsNullOrEmpty(testFramework))
{
logger.SendMessage(TestMessageLevel.Warning, $"No TestRoot or TestFramework specified for '{Path.GetFileName(projectFile)}'.");
continue;
}
var projectHome = Path.GetDirectoryName(projectFile);
var testItems = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
var testFolder = Path.Combine(projectHome, testRoot);
if (!Directory.Exists(testFolder))
{
logger.SendMessage(TestMessageLevel.Warning, $"Test folder path '{testFolder}' doesn't exist.");
continue;
}
// grab all files, we try for .ts files first, and only parse the .js files if we don't find any
foreach (var file in Directory.EnumerateFiles(testFolder, "*.ts", SearchOption.AllDirectories))
{
ProcessFiles(file);
}
if (!testItems.Any())
{
foreach (var file in Directory.EnumerateFiles(testFolder, "*.js", SearchOption.AllDirectories))
{
ProcessFiles(file);
}
}
if (testItems.Any())
{
var testFx = frameworkDiscoverer.GetFramework(testFramework);
if (testFx == null)
{
logger.SendMessage(TestMessageLevel.Warning, $"Ignoring unsupported test framework '{testFramework}'.");
return;
}
var nodeExePath = Nodejs.GetAbsoluteNodeExePath(projectHome, propertyGroup.Descendants(NodeProjectProperty.NodeExePath).FirstOrDefault()?.Value);
if (string.IsNullOrEmpty(nodeExePath))
{
// if nothing specified in the project fallback to environment
nodeExePath = Nodejs.GetPathToNodeExecutableFromEnvironment();
}
this.DiscoverTests(testItems, testFx, discoverySink, logger, nodeExePath, projectFile);
}
void ProcessFiles(string fileAbsolutePath)
{
var typeScriptTest = TypeScript.TypeScriptHelpers.IsTypeScriptFile(fileAbsolutePath);
if (typeScriptTest)
{
fileAbsolutePath = TypeScript.TypeScriptHelpers.GetTypeScriptBackedJavaScriptFile(projectHome, outDir, fileAbsolutePath);
}
else if (!StringComparer.OrdinalIgnoreCase.Equals(Path.GetExtension(fileAbsolutePath), ".js"))
{
return;
}
testItems.Add(fileAbsolutePath);
}
}
}
catch (Exception ex)
{
logger.SendMessage(TestMessageLevel.Error, ex.Message);
throw;
}
}
public void DiscoverTests(ITestCaseDiscoverySink discoverySink, IMessageLogger messageLogger, IDiscoveryContext discoveryContext)
{
var settings = settingsRepository.Load(discoveryContext);
var outputLogger = loggerFactory.CreateOutput(messageLogger, settings);
outputLogger.Info("Discovery started");
IEnumerable <IEnumerable <DiscoveredExample> > groupedSpecifications;
using (var crossDomainLogger = new CrossDomainLogger(outputLogger))
{
groupedSpecifications =
from binaryPath in sources
select binaryTestDiscoverer.Discover(binaryPath, outputLogger, crossDomainLogger);
}
var specifications = groupedSpecifications.SelectMany(group => group);
var testCases = specifications.Select(testCaseMapper.FromDiscoveredExample);
testCases.Do(discoverySink.SendTestCase);
outputLogger.Info("Discovery finished");
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
throw new NotImplementedException();
}
public override void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
Sources = sources;
IsNativeDiscoverTestCalled = true;
base.DiscoverTests(sources, discoveryContext, logger, discoverySink);
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger, ITestCaseDiscoverySink discoverySink)
{
#if LAUNCHDEBUGGER
if (!Debugger.IsAttached)
{
Debugger.Launch();
}
#endif
Initialize(messageLogger);
Info("discovering tests", "started");
// Ensure any channels registered by other adapters are unregistered
CleanUpRegisteredChannels();
foreach (string sourceAssembly in sources)
{
TestLog.SendDebugMessage("Processing " + sourceAssembly);
ITestRunner runner = null;
try
{
runner = GetRunnerFor(sourceAssembly);
try
{
XmlNode loadResult = runner.Load();
// Currently, this will always be the case but it might change
if (loadResult.Name == "test-run")
{
loadResult = loadResult.FirstChild;
}
if (loadResult.GetAttribute("runstate") == "Runnable")
{
XmlNode topNode = runner.Explore(TestFilter.Empty);
using (var testConverter = new TestConverter(TestLog, sourceAssembly))
{
int cases = ProcessTestCases(topNode, discoverySink, testConverter);
TestLog.SendDebugMessage(string.Format("Discovered {0} test cases", cases));
}
}
else
{
var msgNode = loadResult.SelectSingleNode("properties/property[@name='_SKIPREASON']");
if (msgNode != null && msgNode.GetAttribute("value").Contains("contains no tests"))
{
TestLog.SendWarningMessage("Assembly contains no NUnit 3.0 tests: " + sourceAssembly);
}
else
{
TestLog.NUnitLoadError(sourceAssembly);
}
}
}
catch (BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.AssemblyNotSupportedWarning(sourceAssembly);
}
catch (FileNotFoundException ex)
{
// Either the NUnit framework was not referenced by the test assembly
// or some other error occured. Not a problem if not an NUnit assembly.
TestLog.DependentAssemblyNotFoundWarning(ex.FileName, sourceAssembly);
}
catch (FileLoadException ex)
{
// Attempts to load an invalid assembly, or an assembly with missing dependencies
TestLog.LoadingAssemblyFailedWarning(ex.FileName, sourceAssembly);
}
catch (TypeLoadException ex)
{
if (ex.TypeName == "NUnit.Framework.Api.FrameworkController")
{
TestLog.SendWarningMessage(" Skipping NUnit 2.x test assembly");
}
else
{
TestLog.SendErrorMessage("Exception thrown discovering tests in " + sourceAssembly, ex);
}
}
catch (Exception ex)
{
TestLog.SendErrorMessage("Exception thrown discovering tests in " + sourceAssembly, ex);
}
finally
{
if (runner.IsTestRunning)
{
runner.StopRun(true);
}
}
}
finally
{
if (runner != null)
{
runner.Unload();
runner.Dispose();
}
}
}
Info("discovering test", "finished");
Unload();
}
private static Settings GetSettings(IDiscoveryContext runContext)
{
return(Settings.Parse(runContext?.RunSettings?.SettingsXml));
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger, ITestCaseDiscoverySink discoverySink)
{
TestLog.Initialize(messageLogger);
if (RegistryFailure)
{
TestLog.SendErrorMessage(ErrorMsg);
}
Info("discovering tests", "started");
// Ensure any channels registered by other adapters are unregistered
CleanUpRegisteredChannels();
foreach (string sourceAssembly in sources)
{
TestLog.SendDebugMessage("Processing " + sourceAssembly);
TestRunner runner = new TestDomain();
var package = CreateTestPackage(sourceAssembly);
TestConverter testConverter = null;
try
{
if (runner.Load(package))
{
testConverter = new TestConverter(TestLog, sourceAssembly);
int cases = ProcessTestCases(runner.Test, discoverySink, testConverter);
TestLog.SendDebugMessage(string.Format("Discovered {0} test cases", cases));
}
else
{
TestLog.NUnitLoadError(sourceAssembly);
}
}
catch (BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.AssemblyNotSupportedWarning(sourceAssembly);
}
catch (FileNotFoundException ex)
{
// Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
TestLog.DependentAssemblyNotFoundWarning(ex.FileName, sourceAssembly);
}
catch (FileLoadException ex)
{
// Attempts to load an invalid assembly, or an assembly with missing dependencies
TestLog.LoadingAssemblyFailedWarning(ex.FileName, sourceAssembly);
}
catch (UnsupportedFrameworkException ex)
{
TestLog.UnsupportedFrameworkWarning(sourceAssembly);
}
catch (Exception ex)
{
TestLog.SendErrorMessage("Exception thrown discovering tests in " + sourceAssembly, ex);
}
finally
{
if (testConverter != null)
{
testConverter.Dispose();
}
runner.Unload();
}
}
Info("discovering test", "finished");
}
public TestDiscoveryTests(IDiscoveryContext context)
{
_context = context;
}
