/// <summary>
/// Log a message
/// </summary>
/// <param name="message">The message to log</param>
/// <param name="logLevel">The log level</param>
/// <param name="panicMessage">Is it a panic message</param>
public void LogMessage(
string message,
Settings.LoggingLevel logLevel,
bool panicMessage = false)
{
if (logLevel >= _settings?.Logging)
{
_frameworkHandle?.SendMessage(
TestMessageLevel.Informational,
$"[nanoTestAdapter]: {message}");
_logger?.SendMessage(
TestMessageLevel.Informational,
$"[nanoTestAdapter]: {message}");
}
else if (panicMessage)
{
_frameworkHandle?.SendMessage(
TestMessageLevel.Error,
$"[nanoTestAdapter] **PANIC**: {message}");
_logger?.SendMessage(
TestMessageLevel.Error,
$"[nanoTestAdapter] **PANIC**: {message}");
}
}
private void ParseTests(StreamReader standardOutput, ITestCaseDiscoverySink discoverySink, IMessageLogger logger, string source)
{
string testcase = "";
bool testsStarted = false;
Regex testCaseMatch = new Regex(@".*\.$");
while (standardOutput.Peek() != -1)
{
string line = standardOutput.ReadLine();
if (!string.IsNullOrEmpty(line))
{
if (testCaseMatch.IsMatch(line))
{
testsStarted = true;
testcase = line.Trim();
logger.SendMessage(TestMessageLevel.Informational, string.Format("Found test case {0}", testcase.Trim('.')));
}
else if(testsStarted)
{
TestCase test = new TestCase(testcase + line.Trim(), Plugin.VisualStudio2012.GTest.GTestExecutor.ExecutorUri, source);
discoverySink.SendTestCase(test);
logger.SendMessage(TestMessageLevel.Informational, string.Format("Found test {0}", testcase + line.Trim()));
}
}
}
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
ChutzpahTracer.TraceInformation("Begin Test Adapter Discover Tests");
var settingsProvider = discoveryContext.RunSettings.GetSettings(AdapterConstants.SettingsName) as ChutzpahAdapterSettingsProvider;
var settings = settingsProvider != null ? settingsProvider.Settings : new ChutzpahAdapterSettings();
ChutzpahTracingHelper.Toggle(settings.EnabledTracing);
var testOptions = new TestOptions
{
MaxDegreeOfParallelism = settings.MaxDegreeOfParallelism,
ChutzpahSettingsFileEnvironments = new ChutzpahSettingsFileEnvironments(settings.ChutzpahSettingsFileEnvironments)
};
IList<TestError> errors;
var testCases = testRunner.DiscoverTests(sources, testOptions, out errors);
ChutzpahTracer.TraceInformation("Sending discovered tests to test case discovery sink");
foreach (var testCase in testCases)
{
var vsTestCase = testCase.ToVsTestCase();
discoverySink.SendTestCase(vsTestCase);
}
foreach (var error in errors)
{
logger.SendMessage(TestMessageLevel.Error, RunnerCallback.FormatFileErrorMessage(error));
}
ChutzpahTracer.TraceInformation("End Test Adapter Discover Tests");
}
public static IEnumerable <TestCase> GetTests(IEnumerable <string> sourceFiles, ITestCaseDiscoverySink discoverySink, IMessageLogger logger)
{
var tests = new List <TestCase>();
Parallel.ForEach(sourceFiles, s =>
{
var parser = new Parser();
Feature feature;
try
{
using (var reader = File.OpenText(s))
{
logger?.SendMessage(TestMessageLevel.Informational, $"Parsing: {s}");
feature = parser.Parse(reader);
logger?.SendMessage(TestMessageLevel.Informational, $"Parsed: {s}");
}
foreach (var scenario in feature.ScenarioDefinitions)
{
logger?.SendMessage(TestMessageLevel.Error, $"Found scenario '{scenario.Name}' in '{feature.Name}'");
var testCase = new TestCase(feature.Name + "." + scenario.Name, CucumberJsTestExecutor.ExecutorUri, s)
{
CodeFilePath = s,
DisplayName = scenario.Name,
LineNumber = scenario.Location.Line,
};
if (discoverySink != null)
{
discoverySink.SendTestCase(testCase);
}
tests.Add(testCase);
}
}
catch (Exception e)
{
logger?.SendMessage(TestMessageLevel.Error, $"Error parsing '{s}': {e.Message} {e.StackTrace}");
}
});
return(tests);
}
public static IEnumerable<NodeUnitTestDescriptor> DiscoverTests(string filename, IMessageLogger logger)
{
List<NodeUnitTestDescriptor> foundTests = new List<NodeUnitTestDescriptor>();
StringBuilder sbException = new StringBuilder();
Process proc = new Process();
string nodeFullPath = NodeJsHelper.LocateNodeJs();
proc.StartInfo.FileName = nodeFullPath;
proc.StartInfo.UseShellExecute = false;
proc.StartInfo.RedirectStandardInput = true;
proc.StartInfo.RedirectStandardOutput = true;
proc.StartInfo.RedirectStandardError = true;
proc.StartInfo.CreateNoWindow = true;
proc.OutputDataReceived += (sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
try
{
var test = JsonConvert.DeserializeObject<NodeUnitTestDescriptor>(args.Data);
foundTests.Add(test);
}
catch(JsonReaderException)
{
logger.SendMessage(TestMessageLevel.Informational, args.Data);
}
}
};
proc.ErrorDataReceived += (sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
sbException.AppendLine(args.Data);
}
};
proc.Start();
proc.BeginOutputReadLine();
proc.BeginErrorReadLine();
proc.StandardInput.Write(Resources.Disco);
proc.StandardInput.Write("disco(\"" + filename.Replace("\\", "\\\\") + "\");");
proc.StandardInput.Close();
proc.WaitForExit();
if (sbException.Length > 0)
{
throw new Exception(sbException.ToString());
}
return foundTests;
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger,
ITestCaseDiscoverySink discoverySink)
{
logger.SendMessage(TestMessageLevel.Informational, Process.GetCurrentProcess().ProcessName + " Id: " + Process.GetCurrentProcess().Id.ToString());
foreach (var source in sources)
{
logger.SendMessage(TestMessageLevel.Informational, source);
}
try
{
GetTests(sources, discoverySink);
}
catch (Exception e)
{
logger.SendMessage(TestMessageLevel.Error, "exception thrown during test discovery: " + e.Message);
}
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
logger.SendMessage(TestMessageLevel.Informational, "Intent 1.0");
foreach (var p in sources)
{
try
{
var asm = Assembly.LoadFrom(p);
if (asm.IsExcluded())
{
continue;
}
var ts = asm.GetTypes().SkipExcluded();
foreach (var t in ts)
{
var ms = t.GetMethods().SkipExcluded();
foreach (var m in ms)
{
try
{
var discoveredTest = new TestCase(m.Name, VsTestExecutor.Uri, p)
{
DisplayName = m.Name.Replace("_", " ")
};
logger.SendMessage(TestMessageLevel.Informational, $"Found test '{discoveredTest.DisplayName}'.");
discoverySink.SendTestCase(discoveredTest);
}
catch (Exception ex)
{
logger.SendMessage(TestMessageLevel.Error, ex.ToString());
}
}
}
}
catch (Exception ex)
{
logger.SendMessage(TestMessageLevel.Error, ex.ToString());
}
}
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
// indicate start of discovery
logger.SendMessage(TestMessageLevel.Informational, Strings.DISCOVERER_STARTING);
int discoveredSpecCount = 0;
int sourcesWithSpecs = 0;
Settings settings = GetSettings(discoveryContext);
foreach (string assemblyPath in sources.Distinct())
{
try
{
if (!File.Exists(Path.Combine(Path.GetDirectoryName(Path.GetFullPath(assemblyPath)), "Machine.Specifications.dll")))
continue;
sourcesWithSpecs++;
logger.SendMessage(TestMessageLevel.Informational, string.Format(Strings.DISCOVERER_LOOKINGIN, assemblyPath));
List<TestCase> specs = discoverer.DiscoverSpecs(assemblyPath)
.Select(spec => SpecTestHelper.GetVSTestCaseFromMSpecTestCase(assemblyPath, spec, settings.DisableFullTestNameInIDE, MSpecTestAdapter.uri, CreateTrait))
.ToList();
foreach (TestCase discoveredTest in specs)
{
discoveredSpecCount++;
if (discoverySink != null)
{
discoverySink.SendTestCase(discoveredTest);
}
}
}
catch (Exception discoverException)
{
logger.SendMessage(TestMessageLevel.Error, string.Format(Strings.DISCOVERER_ERROR, assemblyPath, discoverException.Message));
}
}
// indicate that we are finished discovering
logger.SendMessage(TestMessageLevel.Informational, string.Format(Strings.DISCOVERER_COMPLETE, discoveredSpecCount, sources.Count(), sourcesWithSpecs));
}
protected override bool Visit(IDiscoveryCompleteMessage discoveryComplete)
{
foreach (var message in discoveryComplete.Warnings)
{
logger.SendMessage(TestMessageLevel.Warning, message);
}
SendExistingTestCases();
return(!cancelThunk());
}
/// <summary>
/// Initializes a new instance of the <see cref="AssemblyLoader" /> class.
/// </summary>
/// <param name="logger">The logger.</param>
public AssemblyLoader(IMessageLogger logger)
{
this.textWriter = new TextWriterLogger(logger);
this.appDomainHelper = new AppDomainHelper();
this.delegateTraceListener = new DelegateTraceListener(
message => logger.SendMessage(TestMessageLevel.Informational, message));
this.proxyDomain = this.appDomainHelper.CreateInstance <ProxyLoader>();
}
GetDiscovererToSourcesMap(string extensionAssembly, IEnumerable <string> sources, IMessageLogger logger)
{
var allDiscoverers = GetDiscoverers(extensionAssembly, throwOnError: true);
if (allDiscoverers == null || !allDiscoverers.Any())
{
// No discoverer available, log a warning
logger.SendMessage(
TestMessageLevel.Warning,
String.Format(CultureInfo.CurrentCulture, CrossPlatEngineResources.NoDiscovererRegistered));
return(null);
}
var result =
new Dictionary <LazyExtension <ITestDiscoverer, ITestDiscovererCapabilities>, IEnumerable <string> >();
var sourcesForWhichNoDiscovererIsAvailable = new List <string>(sources);
foreach (var discoverer in allDiscoverers)
{
// Find the sources which this discoverer can look at.
// Based on whether it is registered for a matching file extension or no file extensions at all.
var matchingSources = (from source in sources
where
(discoverer.Metadata.FileExtension == null ||
discoverer.Metadata.FileExtension.Contains(
Path.GetExtension(source),
StringComparer.OrdinalIgnoreCase))
select source).ToList(); // ToList is required to actually execute the query
// Update the source list for which no matching source is available.
if (matchingSources.Any())
{
sourcesForWhichNoDiscovererIsAvailable =
sourcesForWhichNoDiscovererIsAvailable.Except(matchingSources, StringComparer.OrdinalIgnoreCase)
.ToList();
result.Add(discoverer, matchingSources);
}
}
if (EqtTrace.IsWarningEnabled && sourcesForWhichNoDiscovererIsAvailable != null)
{
foreach (var source in sourcesForWhichNoDiscovererIsAvailable)
{
// Log a warning to logfile, not to the "default logger for discovery time messages".
EqtTrace.Warning(
"No test discoverer is registered to perform discovery for the type of test source '{0}'. Register a test discoverer for this source type and try again.",
source);
}
}
return(result);
}
/// <inheritdoc />
protected override void Write(Severity severity, string message)
{
switch (severity)
{
case Severity.Debug:
_logger.SendMessage(TestMessageLevel.Informational, message);
break;
case Severity.Info:
_logger.SendMessage(TestMessageLevel.Warning, message);
break;
case Severity.Error:
_logger.SendMessage(TestMessageLevel.Error, message);
break;
default:
throw new ArgumentOutOfRangeException(nameof(severity), severity, null);
}
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
// indicate start of discovery
logger.SendMessage(TestMessageLevel.Informational, Strings.DISCOVERER_STARTING);
int discoveredSpecCount = 0;
int sourcesWithSpecs = 0;
foreach (string assemblyPath in sources)
{
try
{
ISpecificationDiscoverer specificationDiscoverer = this.adapterFactory.CreateDiscover();
// only bother discovering if mspec is available and the assembly to san has a reference to mspec
if (specificationDiscoverer.SourceDirectoryContainsMSpec(assemblyPath) && specificationDiscoverer.AssemblyContainsMSpecReference(assemblyPath))
{
sourcesWithSpecs++;
// indicate which assembly we are looking in
logger.SendMessage(TestMessageLevel.Informational, string.Format(Strings.DISCOVERER_LOOKINGIN, assemblyPath));
// do the actual discovery
foreach (TestCase discoveredTest in this.GetTestCases((ISpecificationDiscoverer)specificationDiscoverer, assemblyPath))
{
discoveredSpecCount++;
if (discoverySink != null)
{
discoverySink.SendTestCase(discoveredTest);
}
}
}
}
catch (Exception discoverException)
{
logger.SendMessage(TestMessageLevel.Error, string.Format(Strings.DISCOVERER_ERROR, assemblyPath, discoverException.Message));
}
}
// indicate that we are finished discovering
logger.SendMessage(TestMessageLevel.Informational, string.Format(Strings.DISCOVERER_COMPLETE, discoveredSpecCount, sources.Count(), sourcesWithSpecs));
}
public List<NodejsTestInfo> FindTests(IEnumerable<string> testFiles,
string nodeExe,
IMessageLogger logger,
string workingDirectory) {
string testInfo = string.Empty;
string discoverResultFile = Path.GetTempFileName();
try {
string stdout = EvaluateJavaScript(nodeExe, string.Join(";", testFiles), discoverResultFile, logger, workingDirectory);
if (!String.IsNullOrWhiteSpace(stdout)) {
IEnumerable<String> stdoutLines = stdout.Split(new string[] {Environment.NewLine},
StringSplitOptions.RemoveEmptyEntries).Where(s => s.StartsWith("NTVS_ERROR:")).Select(s => s.Trim().Remove(0,11));
if (stdoutLines != null && stdoutLines.Count() > 0) {
foreach (string s in stdoutLines) {
logger.SendMessage(TestMessageLevel.Error, s);
}
//There was an error during detection, return an empty set
return new List<NodejsTestInfo>();
}
}
for (int i = 0; i < 4; i++) {
try {
testInfo = File.ReadAllText(discoverResultFile);
break;
} catch (IOException) {
//We took an error processing the file. Wait a few and try again
Thread.Sleep(500);
}
}
} finally {
try {
File.Delete(discoverResultFile);
} catch (Exception) { //
//Unable to delete for some reason
//We leave the file behind in this case, its in TEMP so eventually OS will clean up
}
}
List<NodejsTestInfo> testCases = new List<NodejsTestInfo>();
List<DiscoveredTest> discoveredTests = (List<DiscoveredTest>)JsonConvert.DeserializeObject(testInfo, typeof(List<DiscoveredTest>));
if (discoveredTests != null) {
foreach (DiscoveredTest discoveredTest in discoveredTests) {
var line = discoveredTest.Line + 1;
var column = discoveredTest.Column + 1;
NodejsTestInfo test = new NodejsTestInfo(discoveredTest.File, discoveredTest.Test, Name, line, column);
testCases.Add(test);
}
}
return testCases;
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
Dispose();
_testCollection = TestCollection.FromSources(sources);
foreach (var test in _testCollection.Tests)
{
logger.SendMessage(TestMessageLevel.Informational, "Test found: " + test.FullName);
discoverySink.SendTestCase(test.ToTestCase());
}
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
// indicate start of discovery
logger.SendMessage(TestMessageLevel.Informational, Strings.DISCOVERER_STARTING);
int discoveredSpecCount = 0;
int sourcesWithSpecs = 0;
foreach (string assemblyPath in sources)
{
try
{
ISpecificationDiscoverer specificationDiscoverer = this.adapterFactory.CreateDiscover();
// only bother discovering if mspec is available and the assembly to san has a reference to mspec
if (specificationDiscoverer.SourceDirectoryContainsMSpec(assemblyPath) && specificationDiscoverer.AssemblyContainsMSpecReference(assemblyPath))
{
sourcesWithSpecs++;
// indicate which assembly we are looking in
logger.SendMessage(TestMessageLevel.Informational, string.Format(Strings.DISCOVERER_LOOKINGIN, assemblyPath));
// do the actual discovery
foreach (TestCase discoveredTest in this.GetTestCases((ISpecificationDiscoverer)specificationDiscoverer, assemblyPath))
{
discoveredSpecCount++;
if (discoverySink != null)
{
discoverySink.SendTestCase(discoveredTest);
}
}
}
}
catch (Exception discoverException)
{
logger.SendMessage(TestMessageLevel.Error, string.Format(Strings.DISCOVERER_ERROR, assemblyPath, discoverException.Message));
}
}
// indicate that we are finished discovering
logger.SendMessage(TestMessageLevel.Informational, string.Format(Strings.DISCOVERER_COMPLETE, discoveredSpecCount, sources.Count(), sourcesWithSpecs));
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
Guard.ArgumentNotNull("sources", sources);
Guard.ArgumentNotNull("logger", logger);
Guard.ArgumentNotNull("discoverySink", discoverySink);
try
{
RemotingUtility.CleanUpRegisteredChannels();
List<SourceSink> sourceSinks = new List<SourceSink>();
using (AssemblyHelper.SubscribeResolve())
try
{
foreach (string source in sources)
{
try
{
if (cancelled)
break;
if (IsXunitTestAssembly(source))
{
var framework = new Xunit2(SourceInformationProvider, source, configFileName: null, shadowCopy: true);
var sink = new VsDiscoveryVisitor(source, framework, logger, discoverySink, () => cancelled);
sourceSinks.Add(new SourceSink { Framework = framework, Source = source, Sink = sink });
framework.Find(includeSourceInformation: true, messageSink: sink);
}
}
catch (Exception e)
{
logger.SendMessage(TestMessageLevel.Error, String.Format("xUnit.net: Exception discovering tests from {0}: {1}", source, e));
}
}
foreach (var sourceSink in sourceSinks)
sourceSink.Sink.Finished.WaitOne();
}
finally
{
foreach (var sourceSink in sourceSinks)
{
sourceSink.Sink.Dispose();
sourceSink.Framework.Dispose();
}
}
}
catch
{
}
}
private string EvaluateJavaScript(string nodeExePath, string testFile, string discoverResultFile, IMessageLogger logger, string workingDirectory)
{
workingDirectory = workingDirectory.TrimEnd(new char['\\']);
#if DEBUG
var arguments = $"{WrapWithQuotes(this.findTestsScriptFile)} {this.Name} {WrapWithQuotes(testFile)} {WrapWithQuotes(discoverResultFile)} {WrapWithQuotes(workingDirectory)}";
logger.SendMessage(TestMessageLevel.Informational, "Arguments: " + arguments);
#endif
var stdout = string.Empty;
try
{
var process = ProcessOutput.Run(nodeExePath, new[] { this.findTestsScriptFile, this.Name, testFile, discoverResultFile, workingDirectory }, workingDirectory, env: null, visible: false, redirector: new DiscoveryRedirector(logger));
process.Wait();
}
catch (FileNotFoundException e)
{
logger.SendMessage(TestMessageLevel.Error, $"Error starting node.exe.{Environment.NewLine}{e}");
}
return(stdout);
}
public void RunTests(IEnumerable <string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
IMessageLogger logger = frameworkHandle;
logger.SendMessage(TestMessageLevel.Informational, "Intent 1.0");
if (runContext.KeepAlive)
{
frameworkHandle.EnableShutdownAfterTestRun = true;
}
new Runner().Run(sources, new VsLoggerAdapter(frameworkHandle, Uri));
}
private void DiscoverTests(IEnumerable <string> testItems, FrameworkDiscoverer frameworkDiscoverer, ITestCaseDiscoverySink discoverySink, IMessageLogger logger, string nodeExePath, string projectFullPath, string testFrameworkName)
{
var testFramework = frameworkDiscoverer.GetFramework(testFrameworkName);
if (testFramework == null)
{
logger.SendMessage(TestMessageLevel.Warning, $"Ignoring unsupported test framework '{testFrameworkName}'.");
}
var discoverWorker = new TestDiscovererWorker(projectFullPath, nodeExePath);
discoverWorker.DiscoverTests(testItems, testFramework, logger, discoverySink, nameof(NetCoreTestDiscoverer));
}
internal void DiscoverTestFiles(string packageJsonPath, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
logger.SendMessage(TestMessageLevel.Informational, $"Parsing '{packageJsonPath}'.");
var packageJson = PackageJsonFactory.Create(packageJsonPath);
if (string.IsNullOrEmpty(packageJson.TestRoot))
{
logger.SendMessage(TestMessageLevel.Informational, "No vsTestOptions|testRoot specified.");
return;
}
var workingDir = Path.GetDirectoryName(packageJsonPath);
var testFolderPath = Path.Combine(workingDir, packageJson.TestRoot);
if (!Directory.Exists(testFolderPath))
{
logger.SendMessage(TestMessageLevel.Error, $"Testroot '{packageJson.TestRoot}' doesn't exist.");
return;
}
var testFx = default(TestFramework);
foreach (var dep in packageJson.AllDependencies)
{
testFx = this.frameworkDiscoverer.GetFramework(dep.Name);
if (testFx != null)
{
break;
}
}
testFx = testFx ?? this.frameworkDiscoverer.GetFramework(TestFrameworkDirectories.ExportRunnerFrameworkName);
var nodeExePath = Nodejs.GetPathToNodeExecutableFromEnvironment();
var worker = new TestDiscovererWorker(packageJsonPath, nodeExePath);
worker.DiscoverTests(testFolderPath, testFx, logger, discoverySink, nameof(PackageJsonTestDiscoverer));
}
private static void LogWarningOnNoTestsDiscovered(IEnumerable <string> sources, string testCaseFilter, IMessageLogger logger)
{
var sourcesString = string.Join(" ", sources);
// Print warning on no tests.
if (string.IsNullOrEmpty(testCaseFilter) == false)
{
var testCaseFilterToShow = TestCaseFilterDeterminer.ShortenTestCaseFilterIfRequired(testCaseFilter);
logger.SendMessage(
TestMessageLevel.Warning,
string.Format(CrossPlatEngineResources.NoTestsAvailableForGivenTestCaseFilter, testCaseFilterToShow, sourcesString));
}
else
{
logger.SendMessage(
TestMessageLevel.Warning,
string.Format(
CultureInfo.CurrentUICulture,
CrossPlatEngineResources.TestRunFailed_NoDiscovererFound_NoTestsAreAvailableInTheSources,
sourcesString));
}
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext,
IMessageLogger logger,
ITestCaseDiscoverySink discoverySink)
{
var settingsProvider = discoveryContext.RunSettings.GetSettings(GaugeTestRunSettings.SettingsName)
as IGaugeTestRunSettingsService;
if (settingsProvider == null)
{
logger.SendMessage(TestMessageLevel.Error, $"Unable to retrieve Test settings provider for {GaugeTestRunSettings.SettingsName}");
return;
}
GetSpecs(settingsProvider.Settings, discoverySink, sources, logger);
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
// Run each assembly (project) separately
foreach (var source in sources)
{
try
{
ProcessSource(discoverySink, source);
}
catch (System.Exception e)
{
logger.SendMessage(TestMessageLevel.Error, e.Message + "\n" + e.StackTrace);
}
}
}
private static RunSettingsProvider SafeGetRunSettingsProvider(IRunSettings runSettings, IMessageLogger messageLogger)
{
try
{
return(runSettings.GetSettings(GoogleTestConstants.SettingsName) as RunSettingsProvider);
}
catch (Exception e)
{
string errorMessage =
$"ERROR: Visual Studio test framework failed to provide settings. Error message: {e.Message}";
// if fallback settings are configured, we do not want to make the tests fail
var level = AreFallbackSettingsConfigured() ? TestMessageLevel.Informational : TestMessageLevel.Error;
messageLogger.SendMessage(level, errorMessage);
return(null);
}
}
private void DiscoverTests(Dictionary <string, HashSet <string> > testItems, FrameworkDiscoverer frameworkDiscoverer, ITestCaseDiscoverySink discoverySink, IMessageLogger logger, string nodeExePath, string projectFullPath)
{
foreach (var testFx in testItems.Keys)
{
var testFramework = frameworkDiscoverer.GetFramework(testFx);
if (testFramework == null)
{
logger.SendMessage(TestMessageLevel.Warning, $"Ignoring unsupported test framework '{testFx}'.");
continue;
}
var fileList = testItems[testFx];
var discoverWorker = new TestDiscovererWorker(projectFullPath, NodejsConstants.ExecutorUri, nodeExePath);
discoverWorker.DiscoverTests(fileList, testFramework, logger, discoverySink);
}
}
public void DiscoverTests(IEnumerable<string> sources, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
Guard.ArgumentNotNull("sources", sources);
Guard.ArgumentNotNull("logger", logger);
Guard.ArgumentNotNull("discoverySink", discoverySink);
foreach (string source in sources)
try {
if (IsXunitTestAssembly(source))
using (ExecutorWrapper executor = new ExecutorWrapper(source, configFilename: null, shadowCopy: true))
foreach (TestCase testCase in GetTestCases(executor))
discoverySink.SendTestCase(testCase);
}
catch (Exception e) {
logger.SendMessage(TestMessageLevel.Error, String.Format("xUnit.net: Exception discovering tests from {0}: {1}", source, e));
}
}
// ReSharper disable once SuggestBaseTypeForParameter
private static object GetTestContainerInstance(Type type, TestFixture fixture, IMessageLogger logger)
{
logger?.SendMessage(TestMessageLevel.Informational, $"Creating instance for type {type.FullName}");
object instance = null;
try
{
if (type.GetConstructor(new[] { typeof(TestFixture) }) != null)
{
logger?.SendMessage(TestMessageLevel.Informational, $"{type.Name}(TestFixture fixture) {{ ... }}");
instance = Instancing.CreateInstance(type, fixture);
}
else if (type.GetConstructor(new[] { typeof(IServiceCollection) }) != null)
{
logger?.SendMessage(TestMessageLevel.Informational,
$"{type.Name}(IServiceCollection services) {{ ... }}");
instance = Instancing.CreateInstance(type, fixture);
}
else if (type.GetConstructor(new[] { typeof(IServiceProvider) }) != null)
{
logger?.SendMessage(TestMessageLevel.Informational,
$"{type.Name}(IServiceProvider serviceProvider) {{ ... }}");
instance = Instancing.CreateInstance(type, fixture);
}
else if (type.GetConstructor(Type.EmptyTypes) != null)
{
logger?.SendMessage(TestMessageLevel.Informational, $"{type.Name}() {{ ... }}");
instance = Instancing.CreateInstance(type);
}
}
catch (Exception e)
{
logger?.SendMessage(TestMessageLevel.Error, e.ToString());
instance = null;
}
if (instance != null)
{
return(instance);
}
logger?.SendMessage(TestMessageLevel.Error,
"Could not find a suitable constructor for the test containing class");
return(null);
}
private void LogSafe(TestMessageLevel level, string message)
{
if (_settings.TimestampOutput)
{
Utils.TimestampMessage(ref message);
}
if (string.IsNullOrWhiteSpace(message))
{
// Visual Studio 2013 is very picky about empty lines...
// But it accepts an 'INVISIBLE SEPARATOR' (U+2063) :-)
message = "\u2063";
}
lock (Lock)
{
_logger.SendMessage(level, message);
}
}
private void DiscoverTestGroup(
IGrouping <PythonProjectSettings, string> testGroup,
IDiscoveryContext discoveryContext,
IMessageLogger logger,
ITestCaseDiscoverySink discoverySink
)
{
PythonProjectSettings settings = testGroup.Key;
if (settings == null || settings.TestFramework != _frameworkType)
{
return;
}
try {
DiscoverTests(testGroup, settings, logger, discoverySink);
} catch (Exception ex) {
logger.SendMessage(TestMessageLevel.Error, ex.Message);
}
}
/// <summary>
/// Returns ITestCaseFilterExpression for TestProperties supported by adapter.
/// </summary>
/// <param name="runContext">The current context of the run.</param>
/// <param name="testExecutionRecorder">Handler to report test messages/start/end and results.</param>
/// <param name="filterHasError">Indicates that the filter is unsupported/has an error.</param>
/// <returns>A filter expression.</returns>
internal ITestCaseFilterExpression GetFilterExpression(IRunContext runContext, IMessageLogger testExecutionRecorder, out bool filterHasError)
{
filterHasError = false;
ITestCaseFilterExpression filter = null;
if (runContext != null)
{
try
{
filter = runContext.GetTestCaseFilter(this.supportedProperties.Keys, this.PropertyProvider);
}
catch (TestPlatformFormatException ex)
{
filterHasError = true;
testExecutionRecorder.SendMessage(TestMessageLevel.Error, ex.Message);
}
}
return(filter);
}
private static TestCase CreateTestCase(MemberInfo member, string source, IMessageLogger logger)
{
Debug.Assert(member.DeclaringType != null, "member.DeclaringType != null");
var fullyQualifiedName = $"{member.DeclaringType.FullName}.{member.Name}";
logger?.SendMessage(TestMessageLevel.Informational, $"Creating test case for {fullyQualifiedName}");
using var session = new DiaSession(source);
var data = session.GetNavigationData(member.DeclaringType.FullName, member.Name);
var test = new TestCase(fullyQualifiedName, new Uri(TestKitchenTestExecutor.ExecutorUri, UriKind.Absolute), source)
{
CodeFilePath = data.FileName,
LineNumber = data.MinLineNumber + 1,
DisplayName = member.Name.Replace("_", " ")
};
return(test);
}
/// <summary>
/// Returns ITestCaseFilterExpression for TestProperties supported by adapter.
/// </summary>
/// <param name="context">The current context of the run.</param>
/// <param name="logger">Handler to report test messages/start/end and results.</param>
/// <param name="filterHasError">Indicates that the filter is unsupported/has an error.</param>
/// <returns>A filter expression.</returns>
internal ITestCaseFilterExpression GetFilterExpression(IDiscoveryContext context, IMessageLogger logger, out bool filterHasError)
{
filterHasError = false;
ITestCaseFilterExpression filter = null;
if (context != null)
{
try
{
filter = (context is IRunContext) ? this.GetTestCaseFilterFromRunContext(context as IRunContext) : this.GetTestCaseFilterFromDiscoveryContext(context);
}
catch (TestPlatformFormatException ex)
{
filterHasError = true;
logger.SendMessage(TestMessageLevel.Error, ex.Message);
}
}
return(filter);
}
public static void ExitCallBack(
IProcessHelper processHelper,
IMessageLogger messageLogger,
object process,
StringBuilder testHostProcessStdError,
Action <HostProviderEventArgs> onHostExited)
{
var exitCode = 0;
var testHostProcessStdErrorStr = testHostProcessStdError.ToString();
processHelper.TryGetExitCode(process, out exitCode);
if (exitCode != 0)
{
EqtTrace.Error("Test host exited with error: {0}", testHostProcessStdErrorStr);
messageLogger.SendMessage(TestMessageLevel.Error, testHostProcessStdErrorStr);
}
onHostExited(new HostProviderEventArgs(testHostProcessStdErrorStr, exitCode, (process as Process).Id));
}
private void LogSafe(TestMessageLevel level, string message)
{
if (_settings.TimestampOutput)
{
string timestamp = DateTime.Now.ToString("HH:mm:ss.fff", CultureInfo.InvariantCulture);
message = $"{timestamp} - {message ?? ""}";
}
if (string.IsNullOrWhiteSpace(message))
{
// Visual Studio 2013 is very picky about empty lines...
// But it accepts an 'INVISIBLE SEPARATOR' (U+2063) :-)
message = "\u2063";
}
lock (Lock)
{
_logger.SendMessage(level, message);
}
}
private static RunSettingsContainer GetRunSettingsContainer(RunSettingsProvider settingsProvider,
IMessageLogger messageLogger)
{
RunSettingsContainer ourRunSettings;
if (settingsProvider != null)
{
ourRunSettings = settingsProvider.SettingsContainer;
}
else
{
ourRunSettings = GetRunSettingsFromEnvVariable(messageLogger);
if (ourRunSettings == null)
{
messageLogger.SendMessage(TestMessageLevel.Warning, "Warning: Using default settings.");
ourRunSettings = new RunSettingsContainer();
}
}
return(ourRunSettings);
}
/// <summary>
/// Gets filter expression from discovery context.
/// </summary>
/// <param name="context">Discovery context</param>
/// <param name="logger">The logger to log exception messages too.</param>
/// <returns>Filter expression.</returns>
private ITestCaseFilterExpression GetTestCaseFilterFromDiscoveryContext(IDiscoveryContext context, IMessageLogger logger)
{
try
{
// GetTestCaseFilter is present in DiscoveryContext but not in IDiscoveryContext interface.
MethodInfo methodGetTestCaseFilter = context.GetType().GetRuntimeMethod("GetTestCaseFilter", new[] { typeof(IEnumerable <string>), typeof(Func <string, TestProperty>) });
return((ITestCaseFilterExpression)methodGetTestCaseFilter?.Invoke(context, new object[] { this.supportedProperties.Keys, (Func <string, TestProperty>) this.PropertyProvider }));
}
catch (Exception ex)
{
// In case of UWP .Net Native Tool Chain compilation. Invoking methods via Reflection doesn't work, hence discovery always fails.
// Hence throwing exception only if it is of type TargetInvocationException(i.e. Method got invoked but something went wrong in GetTestCaseFilter Method)
if (ex is TargetInvocationException)
{
throw ex.InnerException;
}
logger.SendMessage(TestMessageLevel.Warning, ex.Message);
}
return(null);
}
/// <summary>
/// Method that accepts a message along with the severity level so as to be printed onto the tests output tab
/// </summary>
/// <param name="testMessageLevel">level parameter used to indicate the severity of the message</param>
/// <param name="message">text message that needs to be printed</param>
/// <remarks>In case the Logger is not properly initialized then any messages are simply discarded without raising any exceptions</remarks>
public static void SendMessage(TestMessageLevel testMessageLevel, string message)
{
if (_loggerInstance != null)
{
_loggerInstance.SendMessage(testMessageLevel, message);
}
switch (testMessageLevel)
{
case TestMessageLevel.Informational:
log4netLogger.Info(message);
break;
case TestMessageLevel.Warning:
log4netLogger.Warn(message);
break;
case TestMessageLevel.Error:
log4netLogger.Error(message);
break;
}
}
private void LogSafe(TestMessageLevel level, string message)
{
if (_timeStampOutput())
{
Utils.TimestampMessage(ref message);
}
if (string.IsNullOrWhiteSpace(message))
{
// Visual Studio 2013 is very picky about empty lines...
// But it accepts an 'INVISIBLE SEPARATOR' (U+2063) :-)
message = "\u2063";
}
_logger.SendMessage(level, message);
ReportFinalLogEntry(
new LogEntry
{
Severity = level.GetSeverity(),
Message = message
});
}
private void DiscoverTests(Dictionary<string, List<TestFileEntry>> testItems, MSBuild.Project proj, ITestCaseDiscoverySink discoverySink, IMessageLogger logger) {
List<TestFrameworks.NodejsTestInfo> result = new List<TestFrameworks.NodejsTestInfo>();
var projectHome = Path.GetFullPath(Path.Combine(proj.DirectoryPath, "."));
var projSource = ((MSBuild.Project)proj).FullPath;
var nodeExePath =
Nodejs.GetAbsoluteNodeExePath(
projectHome,
proj.GetPropertyValue(NodejsConstants.NodeExePath));
if (!File.Exists(nodeExePath)) {
logger.SendMessage(TestMessageLevel.Error, String.Format("Node.exe was not found. Please install Node.js before running tests."));
return;
}
int testCount = 0;
foreach (string testFx in testItems.Keys) {
TestFrameworks.TestFramework testFramework = GetTestFrameworkObject(testFx);
if (testFramework == null) {
logger.SendMessage(TestMessageLevel.Warning, String.Format("Ignoring unsupported test framework {0}", testFx));
continue;
}
List<TestFileEntry> fileList = testItems[testFx];
string files = string.Join(";", fileList.Select(p=>p.File));
logger.SendMessage(TestMessageLevel.Informational, String.Format("Processing: {0}", files));
List<TestFrameworks.NodejsTestInfo> discoveredTestCases = testFramework.FindTests(fileList.Select(p => p.File), nodeExePath, logger, projectHome);
testCount += discoveredTestCases.Count;
foreach (TestFrameworks.NodejsTestInfo discoveredTest in discoveredTestCases) {
string qualifiedName = discoveredTest.FullyQualifiedName;
logger.SendMessage(TestMessageLevel.Informational, String.Format(" " /*indent*/ + "Creating TestCase:{0}", qualifiedName));
//figure out the test source info such as line number
string filePath = discoveredTest.ModulePath;
TestFileEntry entry = fileList.Find(p => p.File.Equals(filePath, StringComparison.OrdinalIgnoreCase));
FunctionInformation fi = null;
if (entry.IsTypeScriptTest) {
fi = SourceMapper.MaybeMap(new FunctionInformation(String.Empty,
discoveredTest.TestName,
discoveredTest.SourceLine,
entry.File));
}
discoverySink.SendTestCase(
new TestCase(qualifiedName, TestExecutor.ExecutorUri, projSource) {
CodeFilePath = (fi != null) ? fi.Filename : filePath,
LineNumber = (fi != null && fi.LineNumber.HasValue) ? fi.LineNumber.Value : discoveredTest.SourceLine,
DisplayName = discoveredTest.TestName
});
}
logger.SendMessage(TestMessageLevel.Informational, string.Format("Processing finished for framework of {0}", testFx));
}
if (testCount == 0) {
logger.SendMessage(TestMessageLevel.Warning, String.Format("Discovered 0 testcases."));
}
}
/// <summary>
/// ITestDiscover, Given a list of test sources this method pulls out the test cases
/// </summary>
/// <param name="sources">List of test sources passed from client (Client can be VS or command line)</param>
/// <param name="discoveryContext">Context and runSettings for current run. Discoverer pulls out the tests based on current context</param>
/// <param name="logger">Used to relay messages to registered loggers</param>
/// <param name="discoverySink">Callback used to notify client upon discovery of test cases</param>
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) {
ValidateArg.NotNull(sources, "sources");
ValidateArg.NotNull(discoverySink, "discoverySink");
ValidateArg.NotNull(logger, "logger");
using (var buildEngine = new MSBuild.ProjectCollection()) {
try {
// Load all the test containers passed in (.njsproj msbuild files)
foreach (string source in sources) {
buildEngine.LoadProject(source);
}
foreach (var proj in buildEngine.LoadedProjects) {
var projectHome = Path.GetFullPath(Path.Combine(proj.DirectoryPath, "."));
Dictionary<string, List<TestFileEntry>> testItems = new Dictionary<string,List<TestFileEntry>>(StringComparer.OrdinalIgnoreCase);
// Provide all files to the test analyzer
foreach (var item in proj.Items.Where(item => item.ItemType == "Compile" || item.ItemType == "TypeScriptCompile")) {
//Check to see if this is a TestCase
string value = item.GetMetadataValue("TestFramework");
if (!TestContainerDiscoverer.IsValidTestFramework(value)) {
continue;
}
string fileAbsolutePath = CommonUtils.GetAbsoluteFilePath(projectHome, item.EvaluatedInclude);
bool typeScriptTest = TypeScript.TypeScriptHelpers.IsTypeScriptFile(fileAbsolutePath);
if(typeScriptTest){
fileAbsolutePath = TypeScript.TypeScriptHelpers.GetTypeScriptBackedJavaScriptFile(proj,fileAbsolutePath);
} else if (!Path.GetExtension(fileAbsolutePath).Equals(".js", StringComparison.OrdinalIgnoreCase)) {
continue;
}
List<TestFileEntry> fileList;
if (!testItems.TryGetValue(value, out fileList)){
fileList = new List<TestFileEntry>();
testItems.Add(value, fileList);
}
fileList.Add(new TestFileEntry(fileAbsolutePath,typeScriptTest));
}
//Debug.Fail("Before Discover");
DiscoverTests(testItems, proj, discoverySink, logger);
}
} catch (Exception ex) {
logger.SendMessage(TestMessageLevel.Error, ex.Message);
throw;
} finally {
// Disposing buildEngine does not clear the document cache in
// VS 2013, so manually unload all projects before disposing.
buildEngine.UnloadAllProjects();
}
}
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) {
ValidateArg.NotNull(sources, "sources");
ValidateArg.NotNull(discoverySink, "discoverySink");
var buildEngine = new MSBuild.ProjectCollection();
try {
// Load all the test containers passed in (.pyproj msbuild files)
foreach (string source in sources) {
buildEngine.LoadProject(source);
}
foreach (var proj in buildEngine.LoadedProjects) {
using (var provider = new MSBuildProjectInterpreterFactoryProvider(_interpreterService, proj)) {
try {
provider.DiscoverInterpreters();
} catch (InvalidDataException) {
// This exception can be safely ignored here.
}
var factory = provider.ActiveInterpreter;
if (factory == _interpreterService.NoInterpretersValue) {
if (logger != null) {
logger.SendMessage(TestMessageLevel.Warning, "No interpreters available for project " + proj.FullPath);
}
continue;
}
var projectHome = Path.GetFullPath(Path.Combine(proj.DirectoryPath, proj.GetPropertyValue(PythonConstants.ProjectHomeSetting) ?? "."));
// Do the analysis even if the database is not up to date. At
// worst, we'll get no results.
using (var analyzer = new TestAnalyzer(
factory,
proj.FullPath,
projectHome,
TestExecutor.ExecutorUri
)) {
// Provide all files to the test analyzer
foreach (var item in proj.GetItems("Compile")) {
string fileAbsolutePath = CommonUtils.GetAbsoluteFilePath(projectHome, item.EvaluatedInclude);
string fullName;
try {
fullName = ModulePath.FromFullPath(fileAbsolutePath).ModuleName;
} catch (ArgumentException) {
if (logger != null) {
logger.SendMessage(TestMessageLevel.Warning, "File has an invalid module name: " + fileAbsolutePath);
}
continue;
}
try {
using (var reader = new StreamReader(fileAbsolutePath)) {
analyzer.AddModule(fullName, fileAbsolutePath, reader);
}
} catch (FileNotFoundException) {
// user deleted file, we send the test update, but the project
// isn't saved.
#if DEBUG
} catch (Exception ex) {
if (logger != null) {
logger.SendMessage(TestMessageLevel.Warning, "Failed to discover tests in " + fileAbsolutePath);
logger.SendMessage(TestMessageLevel.Informational, ex.ToString());
}
}
#else
} catch (Exception) {
if (logger != null) {
logger.SendMessage(TestMessageLevel.Warning, "Failed to discover tests in " + fileAbsolutePath);
}
}
#endif
}
public void DiscoverTests(string source, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
foreach (var test in ExternalTestExecutor.GetTestCases(source, s => logger.SendMessage(TestMessageLevel.Error, s)).Select(c => CreateTestCase(source, c)))
discoverySink.SendTestCase(test);
}
private static void SendMessage(TestMessageLevel level, string message, IMessageLogger logger)
{
if (logger != null)
{
logger.SendMessage(level, message);
}
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
_logger = logger;
_logger.SendMessage(TestMessageLevel.Informational, ">>> DiscoverTests");
GetTests(sources, discoverySink);
}
private string EvaluateJavaScript(string nodeExePath, string testFile, string discoverResultFile, IMessageLogger logger, string workingDirectory) {
workingDirectory = workingDirectory.TrimEnd(new char['\\']);
string arguments = WrapWithQuotes(_findTestsScriptFile)
+ " " + Name +
" " + WrapWithQuotes(testFile) +
" " + WrapWithQuotes(discoverResultFile) +
" " + WrapWithQuotes(workingDirectory);
var processStartInfo = new ProcessStartInfo(nodeExePath, arguments);
processStartInfo.CreateNoWindow = true;
processStartInfo.UseShellExecute = false;
processStartInfo.RedirectStandardError = true;
processStartInfo.RedirectStandardOutput = true;
string stdout = String.Empty;
try {
using (var process = Process.Start(processStartInfo)) {
process.EnableRaisingEvents = true;
process.OutputDataReceived += (sender, args) => {
stdout += args.Data + Environment.NewLine;
};
process.ErrorDataReceived += (sender, args) => {
stdout += args.Data + Environment.NewLine;
};
process.BeginErrorReadLine();
process.BeginOutputReadLine();
process.WaitForExit();
#if DEBUG
logger.SendMessage(TestMessageLevel.Informational, String.Format(" Process exited: {0}", process.ExitCode));
#endif
}
#if DEBUG
logger.SendMessage(TestMessageLevel.Informational, String.Format(" StdOut:{0}", stdout));
#endif
} catch (FileNotFoundException e) {
logger.SendMessage(TestMessageLevel.Error, String.Format("Error starting node.exe.\n {0}", e));
}
return stdout;
}
// The Adapter is constructed using the default constructor.
// We don't have any info to initialize it until one of the
// ITestDiscovery or ITestExecutor methods is called. The
// each Discover or Execute method must call this method.
protected virtual void Initialize(IMessageLogger messageLogger)
{
Verbosity = 0; // In case we throw below
AdapterVersion = Assembly.GetExecutingAssembly().GetName().Version.ToString();
try
{
var registry = RegistryCurrentUser.OpenRegistryCurrentUser(@"Software\nunit.org\VSAdapter");
UseVsKeepEngineRunning = registry.Exist("UseVsKeepEngineRunning") && (registry.Read<int>("UseVsKeepEngineRunning") == 1);
ShadowCopy = registry.Exist("ShadowCopy") && (registry.Read<int>("ShadowCopy") == 1);
#if DEBUG && VERBOSE
Verbosity = 1;
#else
Verbosity = (registry.Exist("Verbosity")) ? registry.Read<int>("Verbosity") : 0;
#endif
}
catch (Exception e)
{
messageLogger.SendMessage(TestMessageLevel.Error, "Unable to access registry. Default settings will be used");
messageLogger.SendMessage(TestMessageLevel.Error, e.ToString());
}
TestEngine = new TestEngine();
TestLog = new TestLogger(messageLogger, Verbosity);
}