public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext
, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
#if DEBUG && LAUNCH_DEBUGGER
System.Diagnostics.Debugger.Launch();
#endif
try
{
Logger.Initialize(logger);
var sourceGroups = sources.GroupBy(Path.GetExtension);
foreach (IGrouping<string, string> sourceGroup in sourceGroups)
{
IIutestTestDiscoverer discoverer = iutestTestDiscovererFactory.GetTestDiscoverer(sourceGroup.Key);
if( discoverer != null )
{
discoverer.DiscoverTests(sourceGroup, discoveryContext, logger, discoverySink);
}
}
}
catch( Exception e )
{
Logger.Error("iutest_adapter: Exception caught while DiscoverTests: {0}, {1} ", e.Message, e.HResult );
Logger.Error(e.StackTrace);
}
finally
{
Logger.Uninitialize();
}
}
private void ParseTests(StreamReader standardOutput, ITestCaseDiscoverySink discoverySink, IMessageLogger logger, string source)
{
string testcase = "";
bool testsStarted = false;
Regex testCaseMatch = new Regex(@".*\.$");
while (standardOutput.Peek() != -1)
{
string line = standardOutput.ReadLine();
if (!string.IsNullOrEmpty(line))
{
if (testCaseMatch.IsMatch(line))
{
testsStarted = true;
testcase = line.Trim();
logger.SendMessage(TestMessageLevel.Informational, string.Format("Found test case {0}", testcase.Trim('.')));
}
else if(testsStarted)
{
TestCase test = new TestCase(testcase + line.Trim(), Plugin.VisualStudio2012.GTest.GTestExecutor.ExecutorUri, source);
discoverySink.SendTestCase(test);
logger.SendMessage(TestMessageLevel.Informational, string.Format("Found test {0}", testcase + line.Trim()));
}
}
}
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger log, ITestCaseDiscoverySink discoverySink)
{
log.Version();
RemotingUtility.CleanUpRegisteredChannels();
foreach (var assemblyPath in sources)
{
try
{
if (AssemblyDirectoryContainsFixie(assemblyPath))
{
log.Info("Processing " + assemblyPath);
using (var environment = new ExecutionEnvironment(assemblyPath))
{
var methodGroups = environment.DiscoverTestMethodGroups(new Options());
foreach (var methodGroup in methodGroups)
discoverySink.SendTestCase(new TestCase(methodGroup.FullName, VsTestExecutor.Uri, assemblyPath));
}
}
else
{
log.Info("Skipping " + assemblyPath + " because it is not a test assembly.");
}
}
catch (Exception exception)
{
log.Error(exception);
}
}
}
public static IEnumerable<TestCase> GetTests(IEnumerable<string> sourceFiles, ITestCaseDiscoverySink discoverySink)
{
var tests = new List<TestCase>();
Parallel.ForEach(sourceFiles, s =>
{
var sourceCode = File.ReadAllText(s);
var matches = TestFinderRegex.Matches(sourceCode);
foreach (Match m in matches)
{
var methodParts = m.Groups["Method"].Value.Split(new string[] { ".prototype." }, System.StringSplitOptions.None);
var testClass = methodParts[0];
var testMethod = methodParts[1];
var testName = m.Groups["Name"].Value;
var testDescription = m.Groups["Description"].Value;
var testCase = new TestCase(String.Join(".", methodParts), TSTestExecutor.ExecutorUri, s)
{
CodeFilePath = s,
DisplayName = testName,
};
if (discoverySink != null)
{
discoverySink.SendTestCase(testCase);
}
tests.Add(testCase);
}
});
return tests;
}
public static List<TestCase> GetTests(IEnumerable<string> sources, ITestCaseDiscoverySink discoverySink, IMessageLogger logger = null)
{
var tests = new List<TestCase>();
foreach (var source in sources)
{
var testType = TestType.Pester;
var scriptContents = System.IO.File.ReadAllText(source);
if (scriptContents.StartsWith("#pester", StringComparison.OrdinalIgnoreCase))
{
testType = TestType.Pester;
}
else if (scriptContents.StartsWith("#psate", StringComparison.OrdinalIgnoreCase))
{
testType = TestType.PSate;
}
if (testType == TestType.Pester)
{
DiscoverPesterTests(discoverySink, logger, source, tests);
}
else if (testType == TestType.PSate)
{
DiscoverPsateTests(discoverySink, source, tests);
}
}
return tests;
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, ITestCaseDiscoverySink discoverySink)
{
Code.Require(sources, "sources");
Code.Require(discoverySink, "discoverySink");
if (this.Settings.DiscoveryMethodType == DiscoveryMethodType.DiscoveryListContent)
{
// Delegate to ListContentDiscoverer
ListContentDiscoverer discoverer = new ListContentDiscoverer(new ExternalBoostTestRunnerFactory(), VSProvider);
discoverer.DiscoverTests(sources, discoveryContext, discoverySink);
}
else
{
foreach (string source in sources)
{
TestFramework framework = DiscoverTestFramework(source);
if ((framework != null) && (framework.MasterTestSuite != null))
{
VSDiscoveryVisitor visitor = new VSDiscoveryVisitor(source, discoverySink);
framework.MasterTestSuite.Apply(visitor);
}
}
}
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
testLogger = new TestLogger(logger);
testLogger.SendMainMessage("Discovery started");
foreach (var source in sources)
{
testLogger.SendDebugMessage(String.Format("Processing: '{0}'", source));
try
{
using (var sandbox = new Sandbox<Discoverer>(source))
{
if (sandbox.Content != null)
{
sandbox.Content
.DiscoverTests()
.Select(name => name.ToTestCase(source))
.ForEach(discoverySink.SendTestCase);
}
}
}
catch (Exception ex)
{
testLogger.SendErrorMessage(ex, String.Format("Exception found while discovering tests in source '{0}'", source));
// just go on with the next
}
}
testLogger.SendMainMessage("Discovery finished");
}
public void DiscoverTests(IEnumerable<string> sources,
IDiscoveryContext discoveryContext,
IMessageLogger logger,
ITestCaseDiscoverySink discoverySink)
{
GetTests(sources, discoverySink);
}
private int ProcessTestCases(ITest test, ITestCaseDiscoverySink discoverySink, TestConverter testConverter)
{
int cases = 0;
if (test.IsSuite)
{
foreach (ITest child in test.Tests)
cases += ProcessTestCases(child, discoverySink,testConverter);
}
else
{
try
{
#if LAUNCHDEBUGGER
Debugger.Launch();
#endif
TestCase testCase = testConverter.ConvertTestCase(test);
discoverySink.SendTestCase(testCase);
cases += 1;
}
catch (System.Exception ex)
{
testLog.SendErrorMessage("Exception converting " + test.TestName.FullName, ex);
}
}
return cases;
}
private async Task<int> DiscoverTests(TestSourceSettings settings, ITestLogger logger, ITestCaseDiscoverySink discoverySink)
{
logger = new TestLogger(logger, settings.Name, "Discover");
var count = 0;
var tests = new ConcurrentBag<Guid>();
if (settings.Port > 0)
{
logger.Debug("Start");
var discoverCommand = new DiscoverCommand(settings.Port);
await discoverCommand.Run(spec =>
{
var testCase = CreateTestCase(settings, spec);
tests.Add(testCase.Id);
discoverySink.SendTestCase(testCase);
count += 1;
});
await new RequestRunCommand(settings.Port).Run(tests);
logger.Debug("Complete: {0} tests", count);
}
else
{
logger.Error("Not connected to {0}", TestAdapterInfo.Name);
}
return count;
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
_logger = logger;
_logger.SendMessage(TestMessageLevel.Informational, ">>> DiscoverTests");
GetTests(sources, discoverySink);
}
internal static IEnumerable<TestCase> GetTests(IEnumerable<string> sources, ITestCaseDiscoverySink discoverySink)
{
//if(!Debugger.IsAttached)
// Debugger.Launch();
var tests = new List<TestCase>();
foreach (string source in sources)
{
var TestNames = GetTestNameFromFile(source);
foreach (var testName in TestNames)
{
var normalizedSource = source.ToLowerInvariant();
var testCase = new TestCase(testName.Key, ProtractorTestExecutor.ExecutorUri, normalizedSource);
tests.Add(testCase);
testCase.CodeFilePath = source;
testCase.LineNumber = testName.Value;
if (discoverySink != null)
{
discoverySink.SendTestCase(testCase);
}
}
}
return tests;
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
ChutzpahTracer.TraceInformation("Begin Test Adapter Discover Tests");
var settingsProvider = discoveryContext.RunSettings.GetSettings(AdapterConstants.SettingsName) as ChutzpahAdapterSettingsProvider;
var settings = settingsProvider != null ? settingsProvider.Settings : new ChutzpahAdapterSettings();
ChutzpahTracingHelper.Toggle(settings.EnabledTracing);
var testOptions = new TestOptions
{
MaxDegreeOfParallelism = settings.MaxDegreeOfParallelism,
ChutzpahSettingsFileEnvironments = new ChutzpahSettingsFileEnvironments(settings.ChutzpahSettingsFileEnvironments)
};
IList<TestError> errors;
var testCases = testRunner.DiscoverTests(sources, testOptions, out errors);
ChutzpahTracer.TraceInformation("Sending discovered tests to test case discovery sink");
foreach (var testCase in testCases)
{
var vsTestCase = testCase.ToVsTestCase();
discoverySink.SendTestCase(vsTestCase);
}
foreach (var error in errors)
{
logger.SendMessage(TestMessageLevel.Error, RunnerCallback.FormatFileErrorMessage(error));
}
ChutzpahTracer.TraceInformation("End Test Adapter Discover Tests");
}
public static IEnumerable<TestCase> Discover(IEnumerable<string> sources, ITestCaseDiscoverySink discoverySink)
{
List<TestCase> result = new List<TestCase>();
// System.Diagnostics.Debugger.Launch();
var specList = new List<dynamic>();
foreach (var source in sources)
{
using (var sandbox = new Sandbox<Discover>(source))
{
List<DefinitionSource> discoveredDefinitions = new List<DefinitionSource>();
try
{
discoveredDefinitions = sandbox.Content.DiscoverSpecsFromCurrentAssembly();
}
catch (Exception a)
{
Console.WriteLine(a.Message);
}
discoveredDefinitions
.Select(x => AddToSink(source, x, discoverySink))
.ToList()
.ForEach(x => result.Add(x));
}
}
return result;
}
/// <summary>
/// Finds tests in Catch unit test binaries. Note: We have to run the binary to enumerate tests.
/// </summary>
/// <param name="sources">Binaries to search for tests.</param>
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext context, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
currentLogger = logger;
foreach (var source in sources)
{
switch (checkCatchTestBinaryFormat(source))
{
case CatchBinaryFormat.NoCatchTestBinary:
continue;
case CatchBinaryFormat.CatchTestBinary:
foreach (var testCase in ListTestsInBinary(source))
{
discoverySink.SendTestCase(testCase);
}
continue;
case CatchBinaryFormat.CatchTestBinaryWithXmlTestList:
foreach (var testCase in ListTestsInBinaryXml(source))
{
discoverySink.SendTestCase(testCase);
}
break;
}
}
}
public virtual void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
TestAdapterInfo = CreateTestAdapterInfo();
var testLogger = TestAdapterInfo.CreateLogger(logger);
var discoverLogger = new TestLogger(testLogger, "Discover");
var testSettings = discoveryContext.RunSettings.GetTestSettings(TestAdapterInfo.SettingsName);
var count = 0;
var testCount = 0;
foreach (var source in sources)
{
var sourceSettings = GetSourceSettings(source, testSettings);
if (sourceSettings != null)
{
testCount += DiscoverTests(sourceSettings, testLogger, discoverySink).Result;
count += 1;
}
else
{
discoverLogger.Warn("Could not get settings for {0}", source);
}
}
if (count > 0 || testCount > 0)
{
discoverLogger.Info("{0} tests discovered in {1} test containers", testCount, count);
}
}
internal static IList<TestCase> GetTests(IEnumerable<string> sources, ITestCaseDiscoverySink discoverySink = null, ITestContainer testContainer = null)
{
IList<TestCase> tests = new List<TestCase>();
foreach (var assemblyFileName in sources)
{
Assembly assembly = Assembly.LoadFrom(assemblyFileName);
var methodsWithMoyaAttributes = assembly.GetTypes()
.SelectMany(t => t.GetMethods())
.Where(Reflection.MethodInfoHasMoyaAttribute)
.ToArray();
foreach (MethodInfo methodWithMoyaAttributes in methodsWithMoyaAttributes)
{
var testCase = new TestCase(methodWithMoyaAttributes.Name, Constants.ExecutorUri, assemblyFileName)
{
Id = Guid.NewGuid()
};
tests.Add(testCase);
if (discoverySink != null)
{
discoverySink.SendTestCase(testCase);
}
if (testContainer != null)
{
testContainer.AddTestCaseAndMethod(testCase, methodWithMoyaAttributes);
}
}
}
return tests;
}
public void DiscoverTests(IEnumerable<string> sources
, IDiscoveryContext discoveryContext
, Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging.IMessageLogger logger
, ITestCaseDiscoverySink discoverySink)
{
GetTests(sources, discoverySink);
}
public void DiscoverTests(IEnumerable<string> sources,
IDiscoveryContext discoveryContext,
IMessageLogger logger,
ITestCaseDiscoverySink discoverySink)
{
var validSources = from source in sources
where source.EndsWith(StringHelper.GetSearchExpression(), StringComparison.CurrentCultureIgnoreCase)
select source;
foreach (var source in validSources) {
var results = appDomainRunner.ExecuteSpecifications(source);
var query = from result in results
from @group in result.Examples
from example in @group.Examples
select example;
foreach (var example in query) {
var testCase = new TestCase(example.Reason, DefaultTestExecutor.ExecutorUri, source) {
CodeFilePath = example.FileName,
LineNumber = example.LineNumber
};
discoverySink.SendTestCase(testCase);
}
}
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
Code.Require(sources, "sources");
Code.Require(discoverySink, "discoverySink");
BoostTestAdapterSettings settings = BoostTestAdapterSettingsProvider.GetSettings(discoveryContext);
_sourceFilters = SourceFilterFactory.Get(settings);
IDictionary<string, ProjectInfo> solutioninfo = null;
var numberOfAttempts = 100;
// try several times to overcome "Application is Busy" COMException
while (numberOfAttempts > 0)
{
try
{
solutioninfo = PrepareTestCaseData(sources);
// set numberOfAttempts = 0, because there is no need to try again,
// since obviously no exception was thrown at this point
numberOfAttempts = 0;
}
catch (COMException)
{
--numberOfAttempts;
// re-throw after all attempts have failed
if (numberOfAttempts == 0)
{
throw;
}
}
}
GetBoostTests(solutioninfo, discoverySink);
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) {
ValidateArg.NotNull(sources, "sources");
ValidateArg.NotNull(discoverySink, "discoverySink");
var settings = discoveryContext.RunSettings;
DiscoverTests(sources, logger, discoverySink, settings);
}
/// <summary>
/// @todo add more metadata to test cases (however ctest alone does not provide everything needed)
/// </summary>
/// <param name="sources"></param>
/// <param name="discoveryContext"></param>
/// <param name="logger"></param>
/// <param name="discoverySink"></param>
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging.IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
foreach (var source in sources)
{
var testcase = CTestCase.Parse(source);
if (testcase == null) continue;
discoverySink.SendTestCase(testcase);
}
}
/// <summary>
/// Helper methods which adds a test case to an internal list and sends the test to the discovery sink
/// </summary>
/// <param name="testCase">the test case to be added</param>
/// <param name="discoverySink">the discovery sink where the test case is sent to</param>
public static void AddTestCase(TestCase testCase, ITestCaseDiscoverySink discoverySink)
{
//send to discovery sink
if (null != discoverySink)
{
Logger.Info("Found test: {0}", testCase.FullyQualifiedName);
discoverySink.SendTestCase(testCase);
}
}
public TestGeneratorAdapterTests()
{
testGeneratorDiscoverer = Substitute.For<ITestGeneratorDiscoverer>();
discoveryContext = Substitute.For<IDiscoveryContext>();
discoverySink = Substitute.For<ITestCaseDiscoverySink>();
logger = Substitute.For<IMessageLogger>();
frameworkHandle = Substitute.For<IFrameworkHandle>();
runContext = Substitute.For<IRunContext>();
testGenerator = Substitute.For<ITestGenerator>();
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger, ITestCaseDiscoverySink discoverySink)
{
testLog.Initialize(messageLogger);
Info("discovering tests", "started");
// Ensure any channels registered by other adapters are unregistered
CleanUpRegisteredChannels();
foreach (string sourceAssembly in sources)
{
testLog.SendDebugMessage("Processing " + sourceAssembly);
TestRunner runner = new TestDomain();
TestPackage package = new TestPackage(sourceAssembly);
TestConverter testConverter = null;
try
{
if (runner.Load(package))
{
testConverter = new TestConverter(testLog, sourceAssembly);
int cases = ProcessTestCases(runner.Test, discoverySink, testConverter);
testLog.SendDebugMessage(string.Format("Discovered {0} test cases", cases));
}
else
{
testLog.NUnitLoadError(sourceAssembly);
}
}
catch (System.BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
testLog.AssemblyNotSupportedWarning(sourceAssembly);
}
catch (System.IO.FileNotFoundException ex)
{
// Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
testLog.DependentAssemblyNotFoundWarning(ex.FileName, sourceAssembly);
}
catch (System.Exception ex)
{
testLog.SendErrorMessage("Exception thrown discovering tests in " + sourceAssembly, ex);
}
finally
{
if (testConverter != null)
testConverter.Dispose();
testConverter = null;
runner.Unload();
}
}
Info("discovering test", "finished");
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
foreach(var source in sources) {
var xDomainSink = new TestAdapterLogger(logger, source);
xDomainSink.OnBeginTest += (_, e) => discoverySink.SendTestCase(e.TestCase);
CrossDomainConeRunner.WithProxyInDomain<ConeTestAdapterProxy, int>(string.Empty,
new [] { source, },
proxy => proxy.DiscoverTests(source, xDomainSink)
);
}
}
public void GetBoostTests(IDictionary<string, ProjectInfo> solutionInfo, ITestCaseDiscoverySink discoverySink)
{
if (solutionInfo != null)
{
foreach (KeyValuePair<string, ProjectInfo> info in solutionInfo)
{
string source = info.Key;
ProjectInfo projectInfo = info.Value;
foreach(var sourceFile in projectInfo.CppSourceFiles)
{
try
{
using (var sr = new StreamReader(sourceFile))
{
try
{
var cppSourceFile = new CppSourceFile()
{
FileName = sourceFile,
SourceCode = sr.ReadToEnd()
};
/*
* it is important that the pre-processor defines at project level are not modified
* because every source file in the project has to have the same starting point.
*/
ApplySourceFilter(cppSourceFile, new Defines(projectInfo.DefinesHandler));
//call to cpy ctor
DiscoverBoostTests(cppSourceFile, source, discoverySink);
}
catch (Exception ex)
{
Logger.Error(
"Exception raised while discovering tests from \"{0}\" of project \"{1}\", ({2})",
sourceFile, projectInfo.ProjectExe, ex.Message);
Logger.Error(ex.StackTrace);
}
}
}
catch
{
Logger.Error("Unable to open file \"{0}\" of project \"{1}\".", sourceFile, projectInfo.ProjectExe);
}
}
}
}
else
{
Logger.Error("the solutionInfo object was found to be null whilst");
}
}
/// <summary>
/// ITestDiscover, Given a list of test sources this method pulls out the test cases
/// </summary>
/// <param name="sources">List of test sources passed from client (Client can be VS or command line)</param>
/// <param name="discoveryContext">Context and runSettings for current run. Discoverer pulls out the tests based on current context</param>
/// <param name="logger">Used to relay messages to registered loggers</param>
/// <param name="discoverySink">Callback used to notify client upon discovery of test cases</param>
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) {
ValidateArg.NotNull(sources, "sources");
ValidateArg.NotNull(discoverySink, "discoverySink");
ValidateArg.NotNull(logger, "logger");
using (var buildEngine = new MSBuild.ProjectCollection()) {
try {
// Load all the test containers passed in (.njsproj msbuild files)
foreach (string source in sources) {
buildEngine.LoadProject(source);
}
foreach (var proj in buildEngine.LoadedProjects) {
var projectHome = Path.GetFullPath(Path.Combine(proj.DirectoryPath, "."));
Dictionary<string, List<TestFileEntry>> testItems = new Dictionary<string,List<TestFileEntry>>(StringComparer.OrdinalIgnoreCase);
// Provide all files to the test analyzer
foreach (var item in proj.Items.Where(item => item.ItemType == "Compile" || item.ItemType == "TypeScriptCompile")) {
//Check to see if this is a TestCase
string value = item.GetMetadataValue("TestFramework");
if (!TestContainerDiscoverer.IsValidTestFramework(value)) {
continue;
}
string fileAbsolutePath = CommonUtils.GetAbsoluteFilePath(projectHome, item.EvaluatedInclude);
bool typeScriptTest = TypeScript.TypeScriptHelpers.IsTypeScriptFile(fileAbsolutePath);
if(typeScriptTest){
fileAbsolutePath = TypeScript.TypeScriptHelpers.GetTypeScriptBackedJavaScriptFile(proj,fileAbsolutePath);
} else if (!Path.GetExtension(fileAbsolutePath).Equals(".js", StringComparison.OrdinalIgnoreCase)) {
continue;
}
List<TestFileEntry> fileList;
if (!testItems.TryGetValue(value, out fileList)){
fileList = new List<TestFileEntry>();
testItems.Add(value, fileList);
}
fileList.Add(new TestFileEntry(fileAbsolutePath,typeScriptTest));
}
//Debug.Fail("Before Discover");
DiscoverTests(testItems, proj, discoverySink, logger);
}
} catch (Exception ex) {
logger.SendMessage(TestMessageLevel.Error, ex.Message);
throw;
} finally {
// Disposing buildEngine does not clear the document cache in
// VS 2013, so manually unload all projects before disposing.
buildEngine.UnloadAllProjects();
}
}
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, ITestCaseDiscoverySink discoverySink)
{
Code.Require(sources, "sources");
Code.Require(discoverySink, "discoverySink");
if (this.Settings.DiscoveryMethodType == DiscoveryMethodType.DiscoveryListContent)
{
// Delegate to ListContentDiscoverer
ListContentDiscoverer discoverer = new ListContentDiscoverer(new ExternalBoostTestRunnerFactory(), VSProvider);
discoverer.DiscoverTests(sources, discoveryContext, discoverySink);
}
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
Guard.ArgumentNotNull("sources", sources);
Guard.ArgumentNotNull("logger", logger);
Guard.ArgumentNotNull("discoverySink", discoverySink);
try
{
RemotingUtility.CleanUpRegisteredChannels();
List<SourceSink> sourceSinks = new List<SourceSink>();
using (AssemblyHelper.SubscribeResolve())
try
{
foreach (string source in sources)
{
try
{
if (cancelled)
break;
if (IsXunitTestAssembly(source))
{
var framework = new Xunit2(SourceInformationProvider, source, configFileName: null, shadowCopy: true);
var sink = new VsDiscoveryVisitor(source, framework, logger, discoverySink, () => cancelled);
sourceSinks.Add(new SourceSink { Framework = framework, Source = source, Sink = sink });
framework.Find(includeSourceInformation: true, messageSink: sink);
}
}
catch (Exception e)
{
logger.SendMessage(TestMessageLevel.Error, String.Format("xUnit.net: Exception discovering tests from {0}: {1}", source, e));
}
}
foreach (var sourceSink in sourceSinks)
sourceSink.Sink.Finished.WaitOne();
}
finally
{
foreach (var sourceSink in sourceSinks)
{
sourceSink.Sink.Dispose();
sourceSink.Framework.Dispose();
}
}
}
catch
{
}
}
private void DiscoverTests(Dictionary <string, HashSet <TestFileEntry> > testItems, MSBuild.Project proj, ITestCaseDiscoverySink discoverySink, IMessageLogger logger)
{
var result = new List <NodejsTestInfo>();
var projectHome = Path.GetFullPath(Path.Combine(proj.DirectoryPath, "."));
var projSource = proj.FullPath;
var nodeExePath =
Nodejs.GetAbsoluteNodeExePath(
projectHome,
proj.GetPropertyValue(NodeProjectProperty.NodeExePath));
if (!File.Exists(nodeExePath))
{
logger.SendMessage(TestMessageLevel.Error, "Node.exe was not found. Please install Node.js before running tests.");
return;
}
var testCount = 0;
foreach (var testFx in testItems.Keys)
{
var testFramework = FrameworkDiscoverer.Instance.Get(testFx);
if (testFramework == null)
{
logger.SendMessage(TestMessageLevel.Warning, $"Ignoring unsupported test framework '{testFx}'.");
continue;
}
var fileList = testItems[testFx];
var files = string.Join(";", fileList.Select(p => p.FullPath));
logger.SendMessage(TestMessageLevel.Informational, string.Format(CultureInfo.CurrentCulture, "Processing: {0}", files));
var discoveredTestCases = testFramework.FindTests(fileList.Select(p => p.FullPath), nodeExePath, logger, projectRoot: projectHome);
testCount += discoveredTestCases.Count();
foreach (var discoveredTest in discoveredTestCases)
{
var qualifiedName = discoveredTest.FullyQualifiedName;
const string indent = " ";
logger.SendMessage(TestMessageLevel.Informational, $"{indent}Creating TestCase:{qualifiedName}");
//figure out the test source info such as line number
var filePath = discoveredTest.TestPath;
var entry = fileList.First(p => StringComparer.OrdinalIgnoreCase.Equals(p.FullPath, filePath));
FunctionInformation fi = null;
if (entry.IsTypeScriptTest)
{
fi = SourceMapper.MaybeMap(new FunctionInformation(string.Empty,
discoveredTest.TestName,
discoveredTest.SourceLine,
entry.FullPath));
}
var testcase = new TestCase(qualifiedName, NodejsConstants.ExecutorUri, projSource)
{
CodeFilePath = fi?.Filename ?? filePath,
LineNumber = fi?.LineNumber ?? discoveredTest.SourceLine,
DisplayName = discoveredTest.TestName
};
testcase.SetPropertyValue(JavaScriptTestCaseProperties.TestFramework, testFx);
testcase.SetPropertyValue(JavaScriptTestCaseProperties.NodeExePath, nodeExePath);
testcase.SetPropertyValue(JavaScriptTestCaseProperties.ProjectRootDir, projectHome);
testcase.SetPropertyValue(JavaScriptTestCaseProperties.WorkingDir, projectHome);
testcase.SetPropertyValue(JavaScriptTestCaseProperties.TestFile, filePath);
discoverySink.SendTestCase(testcase);
}
logger.SendMessage(TestMessageLevel.Informational, string.Format(CultureInfo.CurrentCulture, "Processing finished for framework '{0}'.", testFx));
}
}
internal void SendTestCases(string source, IEnumerable <UnitTestElement> testElements, ITestCaseDiscoverySink discoverySink)
{
var shouldCollectSourceInformation = MSTestSettings.RunConfigurationSettings.CollectSourceInformation;
var navigationSessions = new Dictionary <string, object>();
try
{
if (shouldCollectSourceInformation)
{
navigationSessions.Add(source, PlatformServiceProvider.Instance.FileOperations.CreateNavigationSession(source));
}
foreach (var testElement in testElements)
{
object testNavigationSession;
var testCase = testElement.ToTestCase();
if (shouldCollectSourceInformation)
{
string testSource = testElement.TestMethod.DeclaringAssemblyName ?? source;
if (!navigationSessions.TryGetValue(testSource, out testNavigationSession))
{
testNavigationSession = PlatformServiceProvider.Instance.FileOperations.CreateNavigationSession(testSource);
navigationSessions.Add(testSource, testNavigationSession);
}
if (testNavigationSession != null)
{
var className = testElement.TestMethod.DeclaringClassFullName
?? testElement.TestMethod.FullClassName;
var methodName = testElement.TestMethod.Name;
// If it is async test method use compiler generated type and method name for navigation data.
if (!string.IsNullOrEmpty(testElement.AsyncTypeName))
{
className = testElement.AsyncTypeName;
// compiler generated method name is "MoveNext".
methodName = "MoveNext";
}
int minLineNumber;
string fileName;
PlatformServiceProvider.Instance.FileOperations.GetNavigationData(
testNavigationSession,
className,
methodName,
out minLineNumber,
out fileName);
if (!string.IsNullOrEmpty(fileName))
{
testCase.LineNumber = minLineNumber;
testCase.CodeFilePath = fileName;
}
}
}
discoverySink.SendTestCase(testCase);
}
}
finally
{
foreach (object navigationSession in navigationSessions.Values)
{
PlatformServiceProvider.Instance.FileOperations.DisposeNavigationSession(navigationSession);
}
}
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
throw new NotImplementedException();
}
public VsTestFrameworkReporter(ITestCaseDiscoverySink sink, ILogger logger) : this(sink, null, false, logger)
{
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
// indicate start of discovery
logger.SendMessage(TestMessageLevel.Informational, "Machine Specifications Visual Studio Test Adapter - Discovering Specifications.");
int discoveredSpecCount = 0;
int sourcesWithSpecs = 0;
Settings settings = GetSettings(discoveryContext);
foreach (string assemblyPath in sources.Distinct())
{
try
{
#if !NETSTANDARD
if (!File.Exists(Path.Combine(Path.GetDirectoryName(Path.GetFullPath(assemblyPath)), "Machine.Specifications.dll")))
{
continue;
}
#endif
sourcesWithSpecs++;
logger.SendMessage(TestMessageLevel.Informational, string.Format("Machine Specifications Visual Studio Test Adapter - Discovering...looking in {0}", assemblyPath));
List <TestCase> specs = discoverer.DiscoverSpecs(assemblyPath)
.Select(spec => SpecTestHelper.GetVSTestCaseFromMSpecTestCase(assemblyPath, spec, settings.DisableFullTestNameInIDE, MSpecTestAdapter.uri))
.ToList();
foreach (TestCase discoveredTest in specs)
{
discoveredSpecCount++;
if (discoverySink != null)
{
discoverySink.SendTestCase(discoveredTest);
}
}
}
catch (Exception discoverException)
{
logger.SendMessage(TestMessageLevel.Error, string.Format("Machine Specifications Visual Studio Test Adapter - Error while discovering specifications in assembly {0} - {1}", assemblyPath, discoverException.Message));
}
}
// indicate that we are finished discovering
logger.SendMessage(TestMessageLevel.Informational, string.Format("Machine Specifications Visual Studio Test Adapter - Discovery Complete - {0} specifications in {2} of {1} assemblies scanned.", discoveredSpecCount, sources.Count(), sourcesWithSpecs));
}
/// <summary>
/// ITestDiscover, Given a list of test sources this method pulls out the test cases
/// </summary>
/// <param name="sources">List of test sources passed from client (Client can be VS or command line)</param>
/// <param name="logger">Used to relay messages to registered loggers</param>
/// <param name="discoverySink">Callback used to notify client upon discovery of test cases</param>
private void DiscoverTestsCore(IEnumerable <string> sources, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
ValidateArg.NotNull(sources, nameof(sources));
ValidateArg.NotNull(discoverySink, nameof(discoverySink));
ValidateArg.NotNull(logger, nameof(logger));
var env = new Dictionary <string, string>();
var root = Environment.GetEnvironmentVariable("VSINSTALLDIR");
#if DEBUG
logger.SendMessage(TestMessageLevel.Informational, $"VSINSTALLDIR: {root}");
#endif
if (!string.IsNullOrEmpty(root))
{
env["VsInstallRoot"] = root;
env["MSBuildExtensionsPath32"] = Path.Combine(root, "MSBuild");
}
using (var buildEngine = new MSBuild.ProjectCollection(env))
{
try
{
// Load all the test containers passed in (.njsproj msbuild files)
foreach (var source in sources)
{
var cleanPath = source.Trim('\'', '"');
buildEngine.LoadProject(cleanPath);
}
FrameworkDiscoverer frameworkDiscoverer = null;
foreach (var proj in buildEngine.LoadedProjects)
{
var projectHome = Path.GetFullPath(Path.Combine(proj.DirectoryPath, "."));
var testItems = new Dictionary <string, HashSet <string> >(StringComparer.OrdinalIgnoreCase);
var testRoot = proj.GetProperty(NodeProjectProperty.TestRoot)?.EvaluatedValue;
var testFramework = proj.GetProperty(NodeProjectProperty.TestFramework)?.EvaluatedValue;
if (!string.IsNullOrEmpty(testRoot) && string.IsNullOrEmpty(testFramework))
{
logger.SendMessage(TestMessageLevel.Warning, $"TestRoot specified for '{Path.GetFileName(proj.FullPath)}' but no TestFramework.");
}
// Provide all files to the test analyzer
foreach (var item in proj.Items)
{
string testFrameworkName;
string fileAbsolutePath;
if (!string.IsNullOrEmpty(testRoot) && !string.IsNullOrEmpty(testFramework))
{
testFrameworkName = testFramework;
var testRootPath = Path.GetFullPath(Path.Combine(proj.DirectoryPath, testRoot));
try
{
fileAbsolutePath = CommonUtils.GetAbsoluteFilePath(projectHome, item.EvaluatedInclude);
}
catch (ArgumentException)
{
// .Net core projects include invalid paths, ignore them and continue checking the items.
continue;
}
if (!fileAbsolutePath.StartsWith(testRootPath, StringComparison.OrdinalIgnoreCase))
{
continue;
}
}
else
{
//Check to see if this is a TestCase
testFrameworkName = item.GetMetadataValue("TestFramework");
if (!TestFramework.IsValidTestFramework(testFrameworkName))
{
continue;
}
fileAbsolutePath = CommonUtils.GetAbsoluteFilePath(projectHome, item.EvaluatedInclude);
}
var typeScriptTest = TypeScript.TypeScriptHelpers.IsTypeScriptFile(fileAbsolutePath);
if (typeScriptTest)
{
fileAbsolutePath = TypeScript.TypeScriptHelpers.GetTypeScriptBackedJavaScriptFile(proj, fileAbsolutePath);
}
else if (!StringComparer.OrdinalIgnoreCase.Equals(Path.GetExtension(fileAbsolutePath), ".js"))
{
continue;
}
if (!testItems.TryGetValue(testFrameworkName, out var fileList))
{
fileList = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
testItems.Add(testFrameworkName, fileList);
}
fileList.Add(fileAbsolutePath);
}
if (testItems.Any())
{
frameworkDiscoverer = frameworkDiscoverer ?? new FrameworkDiscoverer();
var nodeExePath = Nodejs.GetAbsoluteNodeExePath(projectHome, proj.GetPropertyValue(NodeProjectProperty.NodeExePath));
if (string.IsNullOrEmpty(nodeExePath))
{
// if nothing specified in the project fallback to environment
nodeExePath = Nodejs.GetPathToNodeExecutableFromEnvironment();
}
var spaRoot = proj.GetProperty("SpaRoot")?.EvaluatedValue;
var workDir = Path.GetDirectoryName(proj.FullPath);
if (!String.IsNullOrEmpty(spaRoot))
{
workDir = Path.Combine(workDir, spaRoot);
}
this.DiscoverTests(testItems, frameworkDiscoverer, discoverySink, logger, nodeExePath,
proj.FullPath, workDir);
}
}
}
catch (Exception ex)
{
logger.SendMessage(TestMessageLevel.Error, ex.Message);
throw;
}
finally
{
// Disposing buildEngine does not clear the document cache in
// VS 2013, so manually unload all projects before disposing.
buildEngine.UnloadAllProjects();
}
}
}
public virtual void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
DiscoveryContext = discoveryContext;
MessageLogger = logger;
DiscoverySink = discoverySink;
var testCase = new TestCase("A.C.M", new Uri("executor://dllexecutor"), "A");
discoverySink.SendTestCase(testCase);
}
/// <summary>
/// ITestDiscover, Given a list of test sources this method pulls out the test cases
/// </summary>
/// <param name="sources">List of test sources passed from client (Client can be VS or command line)</param>
/// <param name="discoveryContext">Context and runSettings for current run. Discoverer pulls out the tests based on current context</param>
/// <param name="logger">Used to relay messages to registered loggers</param>
/// <param name="discoverySink">Callback used to notify client upon discovery of test cases</param>
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
ValidateArg.NotNull(sources, "sources");
ValidateArg.NotNull(discoverySink, "discoverySink");
ValidateArg.NotNull(logger, "logger");
var env = new Dictionary <string, string>();
#if DEV15
var root = Environment.GetEnvironmentVariable(NodejsConstants.NodeToolsVsInstallRootEnvironmentVariable);
if (!string.IsNullOrEmpty(root))
{
env["VsInstallRoot"] = root;
env["MSBuildExtensionsPath32"] = Path.Combine(root, "MSBuild");
}
#endif
using (var buildEngine = new MSBuild.ProjectCollection(env)) {
try {
// Load all the test containers passed in (.njsproj msbuild files)
foreach (string source in sources)
{
buildEngine.LoadProject(source);
}
foreach (var proj in buildEngine.LoadedProjects)
{
var projectHome = Path.GetFullPath(Path.Combine(proj.DirectoryPath, "."));
Dictionary <string, List <TestFileEntry> > testItems = new Dictionary <string, List <TestFileEntry> >(StringComparer.OrdinalIgnoreCase);
// Provide all files to the test analyzer
foreach (var item in proj.Items.Where(item => item.ItemType == "Compile" || item.ItemType == "TypeScriptCompile"))
{
//Check to see if this is a TestCase
string value = item.GetMetadataValue("TestFramework");
if (!TestContainerDiscoverer.IsValidTestFramework(value))
{
continue;
}
string fileAbsolutePath = CommonUtils.GetAbsoluteFilePath(projectHome, item.EvaluatedInclude);
bool typeScriptTest = TypeScript.TypeScriptHelpers.IsTypeScriptFile(fileAbsolutePath);
if (typeScriptTest)
{
fileAbsolutePath = TypeScript.TypeScriptHelpers.GetTypeScriptBackedJavaScriptFile(proj, fileAbsolutePath);
}
else if (!Path.GetExtension(fileAbsolutePath).Equals(".js", StringComparison.OrdinalIgnoreCase))
{
continue;
}
List <TestFileEntry> fileList;
if (!testItems.TryGetValue(value, out fileList))
{
fileList = new List <TestFileEntry>();
testItems.Add(value, fileList);
}
fileList.Add(new TestFileEntry(fileAbsolutePath, typeScriptTest));
}
//Debug.Fail("Before Discover");
DiscoverTests(testItems, proj, discoverySink, logger);
}
} catch (Exception ex) {
logger.SendMessage(TestMessageLevel.Error, ex.Message);
throw;
} finally {
// Disposing buildEngine does not clear the document cache in
// VS 2013, so manually unload all projects before disposing.
buildEngine.UnloadAllProjects();
}
}
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
// indicate start of discovery
logger.SendMessage(TestMessageLevel.Informational, Strings.DISCOVERER_STARTING);
int discoveredSpecCount = 0;
int sourcesWithSpecs = 0;
Settings settings = GetSettings(discoveryContext);
foreach (string assemblyPath in sources.Distinct())
{
try
{
if (!File.Exists(Path.Combine(Path.GetDirectoryName(Path.GetFullPath(assemblyPath)), "Machine.Specifications.dll")))
{
continue;
}
sourcesWithSpecs++;
logger.SendMessage(TestMessageLevel.Informational, string.Format(Strings.DISCOVERER_LOOKINGIN, assemblyPath));
List <TestCase> specs = discoverer.DiscoverSpecs(assemblyPath)
.Select(spec => SpecTestHelper.GetVSTestCaseFromMSpecTestCase(assemblyPath, spec, settings.DisableFullTestNameInIDE, MSpecTestAdapter.uri, CreateTrait))
.ToList();
foreach (TestCase discoveredTest in specs)
{
discoveredSpecCount++;
if (discoverySink != null)
{
discoverySink.SendTestCase(discoveredTest);
}
}
}
catch (Exception discoverException)
{
logger.SendMessage(TestMessageLevel.Error, string.Format(Strings.DISCOVERER_ERROR, assemblyPath, discoverException.Message));
}
}
// indicate that we are finished discovering
logger.SendMessage(TestMessageLevel.Informational, string.Format(Strings.DISCOVERER_COMPLETE, discoveredSpecCount, sources.Count(), sourcesWithSpecs));
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
var configuration = new Mocha.BddConfiguration(); //TODO: unhardcode
var tests = GetTests(configuration, sources);
foreach (var test in tests)
{
discoverySink.SendTestCase(test);
}
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger, ITestCaseDiscoverySink discoverySink)
{
#if LAUNCHDEBUGGER
if (!Debugger.IsAttached)
{
Debugger.Launch();
}
#endif
Initialize(discoveryContext, messageLogger);
TestLog.Info($"NUnit Adapter {AdapterVersion}: Test discovery starting");
// Ensure any channels registered by other adapters are unregistered
CleanUpRegisteredChannels();
if (Settings.InProcDataCollectorsAvailable && sources.Count() > 1)
{
TestLog.Error("Unexpected to discover tests in multiple assemblies when InProcDataCollectors specified in run configuration.");
Unload();
return;
}
foreach (string sourceAssembly in sources)
{
string sourceAssemblyPath = Path.IsPathRooted(sourceAssembly) ? sourceAssembly : Path.Combine(Directory.GetCurrentDirectory(), sourceAssembly);
TestLog.Debug("Processing " + sourceAssembly);
if (Settings.DumpXmlTestDiscovery)
{
dumpXml = new DumpXml(sourceAssemblyPath);
}
try
{
var package = CreateTestPackage(sourceAssemblyPath, null);
NUnitEngineAdapter.CreateRunner(package);
var results = NUnitEngineAdapter.Explore();
dumpXml?.AddString(results.AsString());
if (results.IsRunnable)
{
int cases;
using (var testConverter = new TestConverterForXml(TestLog, sourceAssemblyPath, Settings))
{
var timing = new TimingLogger(Settings, TestLog);
cases = ProcessTestCases(results, discoverySink, testConverter);
timing.LogTime("Discovery/Processing/Converting:");
}
TestLog.Debug($"Discovered {cases} test cases");
// Only save if seed is not specified in runsettings
// This allows workaround in case there is no valid
// location in which the seed may be saved.
if (cases > 0 && !Settings.RandomSeedSpecified)
{
Settings.SaveRandomSeed(Path.GetDirectoryName(sourceAssemblyPath));
}
}
else
{
if (results.HasNoNUnitTests)
{
if (Settings.Verbosity > 0)
{
TestLog.Info("Assembly contains no NUnit 3.0 tests: " + sourceAssembly);
}
}
else
{
TestLog.Info("NUnit failed to load " + sourceAssembly);
}
}
}
catch (NUnitEngineException e)
{
if (e.InnerException is BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.Warning("Assembly not supported: " + sourceAssembly);
}
else
{
TestLog.Warning("Exception thrown discovering tests in " + sourceAssembly, e);
}
}
catch (BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.Warning("Assembly not supported: " + sourceAssembly);
}
catch (FileNotFoundException ex)
{
// Either the NUnit framework was not referenced by the test assembly
// or some other error occurred. Not a problem if not an NUnit assembly.
TestLog.Warning("Dependent Assembly " + ex.FileName + " of " + sourceAssembly + " not found. Can be ignored if not an NUnit project.");
}
catch (FileLoadException ex)
{
// Attempts to load an invalid assembly, or an assembly with missing dependencies
TestLog.Warning("Assembly " + ex.FileName + " loaded through " + sourceAssembly + " failed. Assembly is ignored. Correct deployment of dependencies if this is an error.");
}
catch (TypeLoadException ex)
{
if (ex.TypeName == "NUnit.Framework.Api.FrameworkController")
{
TestLog.Warning(" Skipping NUnit 2.x test assembly");
}
else
{
TestLog.Warning("Exception thrown discovering tests in " + sourceAssembly, ex);
}
}
catch (Exception ex)
{
TestLog.Warning("Exception thrown discovering tests in " + sourceAssembly, ex);
}
finally
{
dumpXml?.DumpForDiscovery();
NUnitEngineAdapter?.CloseRunner();
}
}
TestLog.Info($"NUnit Adapter {AdapterVersion}: Test discovery complete");
Unload();
}
public XmlTestCaseReader(ITestCaseDiscoverySink testCaseSink)
{
this.testCaseSink = testCaseSink;
_testCases = new List <TestCase>();
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger, ITestCaseDiscoverySink discoverySink)
{
TestLog.Initialize(messageLogger);
if (RegistryFailure)
{
TestLog.SendErrorMessage(ErrorMsg);
}
Info("discovering tests", "started");
// Ensure any channels registered by other adapters are unregistered
CleanUpRegisteredChannels();
foreach (string sourceAssembly in sources)
{
TestLog.SendDebugMessage("Processing " + sourceAssembly);
TestRunner runner = new TestDomain();
var package = CreateTestPackage(sourceAssembly);
TestConverter testConverter = null;
try
{
if (runner.Load(package))
{
testConverter = new TestConverter(TestLog, sourceAssembly);
int cases = ProcessTestCases(runner.Test, discoverySink, testConverter);
TestLog.SendDebugMessage(string.Format("Discovered {0} test cases", cases));
}
else
{
TestLog.NoNUnit2TestsFoundIn(sourceAssembly);
}
}
catch (BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.AssemblyNotSupportedWarning(sourceAssembly);
}
catch (FileNotFoundException ex)
{
// Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
TestLog.DependentAssemblyNotFoundWarning(ex.FileName, sourceAssembly);
}
catch (FileLoadException ex)
{
// Attempts to load an invalid assembly, or an assembly with missing dependencies
TestLog.LoadingAssemblyFailedWarning(ex.FileName, sourceAssembly);
}
catch (UnsupportedFrameworkException)
{
TestLog.UnsupportedFrameworkWarning(sourceAssembly);
}
catch (Exception ex)
{
TestLog.SendErrorMessage("Exception thrown discovering tests in " + sourceAssembly, ex);
}
finally
{
runner.Unload();
}
}
Info("discovering test", "finished");
}
private void DiscoverTests(Dictionary <string, HashSet <string> > testItems, FrameworkDiscoverer frameworkDiscoverer, ITestCaseDiscoverySink discoverySink, IMessageLogger logger, string nodeExePath, string projectFullPath, string workDir)
{
foreach (var testFx in testItems.Keys)
{
var testFramework = frameworkDiscoverer.GetFramework(testFx);
if (testFramework == null)
{
logger.SendMessage(TestMessageLevel.Warning, $"Ignoring unsupported test framework '{testFx}'.");
continue;
}
var fileList = testItems[testFx];
var discoverWorker = new TestDiscovererWorker(projectFullPath, NodejsConstants.ExecutorUri, nodeExePath, workDir);
discoverWorker.DiscoverTests(fileList, testFramework, logger, discoverySink);
}
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
AssemblyResolver.SetupHandler();
this.DiscoverTestsCore(sources, logger, discoverySink);
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
IsDiscoverTestCalled = true;
Sources = sources;
DiscoveryContext = discoveryContext;
MessageLogger = logger;
DiscoverySink = discoverySink;
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
ChutzpahTracer.TraceInformation("Begin Test Adapter Discover Tests");
var settingsProvider = discoveryContext.RunSettings.GetSettings(AdapterConstants.SettingsName) as ChutzpahAdapterSettingsProvider;
var settings = settingsProvider != null ? settingsProvider.Settings : new ChutzpahAdapterSettings();
ChutzpahTracingHelper.Toggle(settings.EnabledTracing);
var testOptions = new TestOptions
{
MaxDegreeOfParallelism = settings.MaxDegreeOfParallelism,
ChutzpahSettingsFileEnvironments = new ChutzpahSettingsFileEnvironments(settings.ChutzpahSettingsFileEnvironments)
};
IList <TestError> errors;
var testCases = testRunner.DiscoverTests(sources, testOptions, out errors);
ChutzpahTracer.TraceInformation("Sending discovered tests to test case discovery sink");
foreach (var testCase in testCases)
{
var vsTestCase = testCase.ToVsTestCase();
discoverySink.SendTestCase(vsTestCase);
}
foreach (var error in errors)
{
logger.SendMessage(TestMessageLevel.Error, RunnerCallback.FormatFileErrorMessage(error));
}
ChutzpahTracer.TraceInformation("End Test Adapter Discover Tests");
}
public override void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
Sources = sources;
IsNativeDiscoverTestCalled = true;
base.DiscoverTests(sources, discoveryContext, logger, discoverySink);
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger, ITestCaseDiscoverySink discoverySink)
{
#if LAUNCHDEBUGGER
if (!Debugger.IsAttached)
{
Debugger.Launch();
}
#endif
Initialize(discoveryContext, messageLogger);
TestLog.Info(string.Format("NUnit Adapter {0}: Test discovery starting", AdapterVersion));
// Ensure any channels registered by other adapters are unregistered
CleanUpRegisteredChannels();
foreach (string sourceAssembly in sources)
{
TestLog.Debug("Processing " + sourceAssembly);
ITestRunner runner = null;
runner = GetRunnerFor(sourceAssembly);
try
{
XmlNode loadResult = runner.Load();
// Currently, this will always be the case but it might change
if (loadResult.Name == "test-run")
{
loadResult = loadResult.FirstChild;
}
if (loadResult.GetAttribute("runstate") == "Runnable")
{
XmlNode topNode = runner.Explore(TestFilter.Empty);
using (var testConverter = new TestConverter(TestLog, sourceAssembly))
{
int cases = ProcessTestCases(topNode, discoverySink, testConverter);
TestLog.Debug(string.Format("Discovered {0} test cases", cases));
}
}
else
{
var msgNode = loadResult.SelectSingleNode("properties/property[@name='_SKIPREASON']");
if (msgNode != null && (new[] { "contains no tests", "Has no TestFixtures" }).Any(msgNode.GetAttribute("value").Contains))
{
TestLog.Info("Assembly contains no NUnit 3.0 tests: " + sourceAssembly);
}
else
{
TestLog.Info("NUnit failed to load " + sourceAssembly);
}
}
}
catch (BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.Warning("Assembly not supported: " + sourceAssembly);
}
catch (FileNotFoundException ex)
{
// Either the NUnit framework was not referenced by the test assembly
// or some other error occured. Not a problem if not an NUnit assembly.
TestLog.Warning("Dependent Assembly " + ex.FileName + " of " + sourceAssembly + " not found. Can be ignored if not a NUnit project.");
}
catch (FileLoadException ex)
{
// Attempts to load an invalid assembly, or an assembly with missing dependencies
TestLog.Warning("Assembly " + ex.FileName + " loaded through " + sourceAssembly + " failed. Assembly is ignored. Correct deployment of dependencies if this is an error.");
}
catch (TypeLoadException ex)
{
if (ex.TypeName == "NUnit.Framework.Api.FrameworkController")
{
TestLog.Warning(" Skipping NUnit 2.x test assembly");
}
else
{
TestLog.Error("Exception thrown discovering tests in " + sourceAssembly, ex);
}
}
catch (Exception ex)
{
TestLog.Error("Exception thrown discovering tests in " + sourceAssembly, ex);
}
finally
{
if (runner.IsTestRunning)
{
runner.StopRun(true);
}
runner.Unload();
runner.Dispose();
}
}
TestLog.Info(string.Format("NUnit Adapter {0}: Test discovery complete", AdapterVersion));
Unload();
}
private void DiscoverTests(Dictionary <string, List <TestFileEntry> > testItems, MSBuild.Project proj, ITestCaseDiscoverySink discoverySink, IMessageLogger logger)
{
List <TestFrameworks.NodejsTestInfo> result = new List <TestFrameworks.NodejsTestInfo>();
var projectHome = Path.GetFullPath(Path.Combine(proj.DirectoryPath, "."));
var projSource = ((MSBuild.Project)proj).FullPath;
var nodeExePath =
Nodejs.GetAbsoluteNodeExePath(
projectHome,
proj.GetPropertyValue(NodejsConstants.NodeExePath));
if (!File.Exists(nodeExePath))
{
logger.SendMessage(TestMessageLevel.Error, String.Format("Node.exe was not found. Please install Node.js before running tests."));
return;
}
int testCount = 0;
foreach (string testFx in testItems.Keys)
{
TestFrameworks.TestFramework testFramework = GetTestFrameworkObject(testFx);
if (testFramework == null)
{
logger.SendMessage(TestMessageLevel.Warning, String.Format("Ignoring unsupported test framework {0}", testFx));
continue;
}
List <TestFileEntry> fileList = testItems[testFx];
string files = string.Join(";", fileList.Select(p => p.File));
logger.SendMessage(TestMessageLevel.Informational, String.Format("Processing: {0}", files));
List <TestFrameworks.NodejsTestInfo> discoveredTestCases = testFramework.FindTests(fileList.Select(p => p.File), nodeExePath, logger, projectHome);
testCount += discoveredTestCases.Count;
foreach (TestFrameworks.NodejsTestInfo discoveredTest in discoveredTestCases)
{
string qualifiedName = discoveredTest.FullyQualifiedName;
logger.SendMessage(TestMessageLevel.Informational, String.Format(" " /*indent*/ + "Creating TestCase:{0}", qualifiedName));
//figure out the test source info such as line number
string filePath = discoveredTest.ModulePath;
TestFileEntry entry = fileList.Find(p => p.File.Equals(filePath, StringComparison.OrdinalIgnoreCase));
FunctionInformation fi = null;
if (entry.IsTypeScriptTest)
{
fi = SourceMapper.MaybeMap(new FunctionInformation(String.Empty,
discoveredTest.TestName,
discoveredTest.SourceLine,
entry.File));
}
discoverySink.SendTestCase(
new TestCase(qualifiedName, TestExecutor.ExecutorUri, projSource)
{
CodeFilePath = (fi != null) ? fi.Filename : filePath,
LineNumber = (fi != null && fi.LineNumber.HasValue) ? fi.LineNumber.Value : discoveredTest.SourceLine,
DisplayName = discoveredTest.TestName
});
}
logger.SendMessage(TestMessageLevel.Informational, string.Format("Processing finished for framework of {0}", testFx));
}
if (testCount == 0)
{
logger.SendMessage(TestMessageLevel.Warning, String.Format("Discovered 0 testcases."));
}
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, ITestCaseDiscoverySink discoverySink)
{
Code.Require(sources, "sources");
Code.Require(discoverySink, "discoverySink");
// Populate loop-invariant attributes and settings
BoostTestAdapterSettings settings = BoostTestAdapterSettingsProvider.GetSettings(discoveryContext);
BoostTestRunnerSettings runnerSettings = new BoostTestRunnerSettings()
{
Timeout = settings.DiscoveryTimeoutMilliseconds
};
BoostTestRunnerCommandLineArgs args = new BoostTestRunnerCommandLineArgs()
{
ListContent = ListContentFormat.DOT
};
foreach (var source in sources)
{
try
{
using (new Utility.TimedScope("Extract settings from Visual Studio for \"{0}\"", source))
{
var vs = _vsProvider?.Instance;
if (vs != null)
{
Logger.Debug("Connected to Visual Studio {0} instance", vs.Version);
}
args.SetWorkingEnvironment(source, settings, vs);
}
}
catch (ROTException ex)
{
Logger.Exception(ex, "Could not retrieve WorkingDirectory from Visual Studio Configuration");
}
catch (COMException ex)
{
Logger.Exception(ex, "Could not retrieve WorkingDirectory from Visual Studio Configuration");
}
try
{
IBoostTestRunner runner = _factory.GetRunner(source, settings.TestRunnerFactoryOptions);
using (TemporaryFile output = new TemporaryFile(TestPathGenerator.Generate(source, ".list.content.gv")))
{
// --list_content output is redirected to standard error
args.StandardErrorFile = output.Path;
Logger.Debug("list_content file: {0}", args.StandardErrorFile);
int resultCode = EXIT_SUCCESS;
using (new Utility.TimedScope("Execute \"--list_content=DOT\" for \"{0}\"", source))
{
using (var context = new DefaultProcessExecutionContext())
{
resultCode = runner.Execute(args, runnerSettings, context);
}
}
// Skip sources for which the --list_content file is not available
if (!File.Exists(args.StandardErrorFile))
{
Logger.Error("--list_content=DOT output for {0} is not available. Skipping.", source);
continue;
}
// If the executable failed to exit with an EXIT_SUCCESS code, skip source and notify user accordingly
if (resultCode != EXIT_SUCCESS)
{
Logger.Error("--list_content=DOT for {0} failed with exit code {1}. Skipping.", source, resultCode);
continue;
}
// Parse --list_content=DOT output
Stream stream = null;
try
{
stream = File.OpenRead(args.StandardErrorFile);
using (var reader = new StreamReader(stream, System.Text.Encoding.Default))
{
stream = null;
TestFrameworkDOTDeserialiser deserialiser = new TestFrameworkDOTDeserialiser(source);
TestFramework framework = deserialiser.Deserialise(reader);
if ((framework != null) && (framework.MasterTestSuite != null))
{
framework.MasterTestSuite.Apply(new VSDiscoveryVisitor(source, GetVersion(runner), discoverySink));
}
}
}
finally
{
if (stream != null)
{
stream.Dispose();
}
}
}
}
catch (Exception ex)
{
Logger.Exception(ex, "Exception caught while discovering tests for {0} ({1} - {2})", source, ex.Message, ex.HResult);
}
}
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
if (sources == null)
{
throw new ArgumentNullException(nameof(sources));
}
if (discoverySink == null)
{
throw new ArgumentNullException(nameof(discoverySink));
}
var settings = discoveryContext.RunSettings;
DiscoverTests(sources, logger, discoverySink, settings);
}
static void DiscoverTests(IEnumerable <string> sources, IMessageLogger logger, ITestCaseDiscoverySink discoverySink, IRunSettings settings)
{
var sourcesSet = new HashSet <string>(sources, StringComparer.OrdinalIgnoreCase);
var executorUri = new Uri(PythonConstants.TestExecutorUriString);
// Test list is sent to us via our run settings which we use to smuggle the
// data we have in our analysis process.
var doc = Read(settings.SettingsXml);
foreach (var t in TestReader.ReadTests(doc, sourcesSet, m => {
logger?.SendMessage(TestMessageLevel.Warning, m);
}))
{
var tc = new TestCase(t.FullyQualifiedName, executorUri, t.SourceFile)
{
DisplayName = t.DisplayName,
LineNumber = t.LineNo,
CodeFilePath = t.FileName
};
discoverySink.SendTestCase(tc);
}
}
