void RunTests(IRunContext runContext, IFrameworkHandle frameworkHandle, LoggerHelper logger, Func <List <AssemblyRunInfo> > testCaseAccessor) { Guard.ArgumentNotNull("runContext", runContext); Guard.ArgumentNotNull("frameworkHandle", frameworkHandle); try { RemotingUtility.CleanUpRegisteredChannels(); cancelled = false; var assemblies = testCaseAccessor(); var parallelizeAssemblies = assemblies.All(runInfo => runInfo.Configuration.ParallelizeAssemblyOrDefault); var reporterMessageHandler = new DefaultRunnerReporter().CreateMessageHandler(new VisualStudioRunnerLogger(logger)); using (AssemblyHelper.SubscribeResolve()) if (parallelizeAssemblies) { assemblies .Select(runInfo => RunTestsInAssemblyAsync(frameworkHandle, logger, reporterMessageHandler, runInfo)) .ToList() .ForEach(@event => @event.WaitOne()); } else { assemblies .ForEach(runInfo => RunTestsInAssembly(frameworkHandle, logger, reporterMessageHandler, runInfo)); } } catch (Exception ex) { logger.LogError("Catastrophic failure: {0}", ex); } }
public override bool Execute() { RemotingUtility.CleanUpRegisteredChannels(); if (TeamCity) { Log.LogError("The 'TeamCity' property is deprecated. Please set the 'Reporter' property to 'teamcity' instead."); return false; } if (Verbose) { Log.LogError("The 'Verbose' property is deprecated. Please set the 'Reporter' property to 'verbose' instead."); return false; } XElement assembliesElement = null; var environment = string.Format("{0}-bit .NET {1}", IntPtr.Size * 8, Environment.Version); if (NeedsXml) assembliesElement = new XElement("assemblies"); switch (MaxParallelThreads) { case null: case "default": break; case "unlimited": maxThreadCount = 0; break; default: int threadValue; if (!int.TryParse(MaxParallelThreads, out threadValue) || threadValue < 0) { Log.LogError("MaxParallelThreads value '{0}' is invalid: must be 'default', 'unlimited', or a positive number", MaxParallelThreads); return false; } maxThreadCount = threadValue; break; } var originalWorkingFolder = Directory.GetCurrentDirectory(); using (AssemblyHelper.SubscribeResolve()) { var reporters = GetAvailableRunnerReporters(); var reporter = reporters.FirstOrDefault(r => r.IsEnvironmentallyEnabled); if (reporter == null && !string.IsNullOrWhiteSpace(Reporter)) { reporter = reporters.FirstOrDefault(r => string.Equals(r.RunnerSwitch, Reporter, StringComparison.OrdinalIgnoreCase)); if (reporter == null) { var switchableReporters = reporters.Where(r => !string.IsNullOrWhiteSpace(r.RunnerSwitch)).Select(r => r.RunnerSwitch.ToLowerInvariant()).OrderBy(x => x).ToList(); if (switchableReporters.Count == 0) Log.LogError("Reporter value '{0}' is invalid. There are no available reporters.", Reporter); else Log.LogError("Reporter value '{0}' is invalid. Available reporters: {1}", Reporter, string.Join(", ", switchableReporters)); return false; } } if (reporter == null) reporter = new DefaultRunnerReporter(); logger = new MSBuildLogger(Log); reporterMessageHandler = reporter.CreateMessageHandler(logger); if (!NoLogo) Log.LogMessage(MessageImportance.High, "xUnit.net MSBuild Runner ({0})", environment); var project = new XunitProject(); foreach (var assembly in Assemblies) { var assemblyFileName = assembly.GetMetadata("FullPath"); var configFileName = assembly.GetMetadata("ConfigFile"); if (configFileName != null && configFileName.Length == 0) configFileName = null; project.Add(new XunitProjectAssembly { AssemblyFilename = assemblyFileName, ConfigFilename = configFileName, ShadowCopy = ShadowCopy }); } if (WorkingFolder != null) Directory.SetCurrentDirectory(WorkingFolder); var clockTime = Stopwatch.StartNew(); if (!parallelizeAssemblies.HasValue) parallelizeAssemblies = project.All(assembly => assembly.Configuration.ParallelizeAssemblyOrDefault); if (parallelizeAssemblies.GetValueOrDefault()) { var tasks = project.Assemblies.Select(assembly => Task.Run(() => ExecuteAssembly(assembly))); var results = Task.WhenAll(tasks).GetAwaiter().GetResult(); foreach (var assemblyElement in results.Where(result => result != null)) assembliesElement.Add(assemblyElement); } else { foreach (var assembly in project.Assemblies) { var assemblyElement = ExecuteAssembly(assembly); if (assemblyElement != null) assembliesElement.Add(assemblyElement); } } clockTime.Stop(); if (completionMessages.Count > 0) reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, completionMessages.OrderBy(kvp => kvp.Key).ToList())); } Directory.SetCurrentDirectory(WorkingFolder ?? originalWorkingFolder); if (NeedsXml) { if (Xml != null) assembliesElement.Save(Xml.GetMetadata("FullPath")); if (XmlV1 != null) Transform("xUnit1.xslt", assembliesElement, XmlV1); if (Html != null) Transform("HTML.xslt", assembliesElement, Html); if (NUnit != null) Transform("NUnitXml.xslt", assembliesElement, NUnit); } return ExitCode == 0; }
protected XunitProject Parse(Predicate <string> fileExists) { var assemblies = new List <Tuple <string, string> >(); Reporter = new DefaultRunnerReporter(); while (arguments.Count > 0) { if (arguments.Peek().StartsWith("-", StringComparison.Ordinal)) { break; } var assemblyFile = arguments.Pop(); if (IsConfigFile(assemblyFile)) { throw new ArgumentException($"expecting assembly, got config file: {assemblyFile}"); } if (!fileExists(assemblyFile)) { throw new ArgumentException($"file not found: {assemblyFile}"); } string configFile = null; if (arguments.Count > 0) { var value = arguments.Peek(); if (!value.StartsWith("-", StringComparison.Ordinal) && IsConfigFile(value)) { configFile = arguments.Pop(); if (!fileExists(configFile)) { throw new ArgumentException($"config file not found: {configFile}"); } } } assemblies.Add(Tuple.Create(assemblyFile, configFile)); } if (assemblies.Count == 0) { throw new ArgumentException("must specify at least one assembly"); } var project = GetProjectFile(assemblies); while (arguments.Count > 0) { var option = PopOption(arguments); var optionName = option.Key.ToLowerInvariant(); if (!optionName.StartsWith("-", StringComparison.Ordinal)) { throw new ArgumentException($"unknown command line option: {option.Key}"); } optionName = optionName.Substring(1); if (optionName == "installlocation") { if (option.Value == null) { throw new ArgumentException("missing argument for -installlocation"); } InstallLocation = option.Value; } else if (optionName == "nologo") { GuardNoOptionValue(option); NoLogo = true; } else if (optionName == "failskips") { GuardNoOptionValue(option); FailSkips = true; } else if (optionName == "nocolor") { GuardNoOptionValue(option); NoColor = true; } else if (optionName == "noappdomain") { GuardNoOptionValue(option); NoAppDomain = true; } else if (optionName == "debug") { GuardNoOptionValue(option); Debug = true; } else if (optionName == "serialize") { GuardNoOptionValue(option); Serialize = true; } else if (optionName == "wait") { GuardNoOptionValue(option); Wait = true; } else if (optionName == "diagnostics") { GuardNoOptionValue(option); DiagnosticMessages = true; } else if (optionName == "maxthreads") { if (option.Value == null) { throw new ArgumentException("missing argument for -maxthreads"); } switch (option.Value) { case "default": MaxParallelThreads = 0; break; case "unlimited": MaxParallelThreads = -1; break; default: int threadValue; if (!int.TryParse(option.Value, out threadValue) || threadValue < 1) { throw new ArgumentException("incorrect argument value for -maxthreads (must be 'default', 'unlimited', or a positive number)"); } MaxParallelThreads = threadValue; break; } } else if (optionName == "parallel") { if (option.Value == null) { throw new ArgumentException("missing argument for -parallel"); } ParallelismOption parallelismOption; if (!Enum.TryParse <ParallelismOption>(option.Value, out parallelismOption)) { throw new ArgumentException("incorrect argument value for -parallel"); } switch (parallelismOption) { case ParallelismOption.all: ParallelizeAssemblies = true; ParallelizeTestCollections = true; break; case ParallelismOption.assemblies: ParallelizeAssemblies = true; ParallelizeTestCollections = false; break; case ParallelismOption.collections: ParallelizeAssemblies = false; ParallelizeTestCollections = true; break; default: ParallelizeAssemblies = false; ParallelizeTestCollections = false; break; } } else if (optionName == "noshadow") { GuardNoOptionValue(option); foreach (var assembly in project.Assemblies) { assembly.Configuration.ShadowCopy = false; } } else if (optionName == "trait") { if (option.Value == null) { throw new ArgumentException("missing argument for -trait"); } var pieces = option.Value.Split('='); if (pieces.Length != 2 || string.IsNullOrEmpty(pieces[0]) || string.IsNullOrEmpty(pieces[1])) { throw new ArgumentException("incorrect argument format for -trait (should be \"name=value\")"); } var name = pieces[0]; var value = pieces[1]; project.Filters.IncludedTraits.Add(name, value); } else if (optionName == "notrait") { if (option.Value == null) { throw new ArgumentException("missing argument for -notrait"); } var pieces = option.Value.Split('='); if (pieces.Length != 2 || string.IsNullOrEmpty(pieces[0]) || string.IsNullOrEmpty(pieces[1])) { throw new ArgumentException("incorrect argument format for -notrait (should be \"name=value\")"); } var name = pieces[0]; var value = pieces[1]; project.Filters.ExcludedTraits.Add(name, value); } else if (optionName == "class") { if (option.Value == null) { throw new ArgumentException("missing argument for -class"); } project.Filters.IncludedClasses.Add(option.Value); } else if (optionName == "method") { if (option.Value == null) { throw new ArgumentException("missing argument for -method"); } project.Filters.IncludedMethods.Add(option.Value); } else if (optionName == "namespace") { if (option.Value == null) { throw new ArgumentException("missing argument for -namespace"); } project.Filters.IncludedNameSpaces.Add(option.Value); } else if (optionName == "xml") { if (option.Value == null) { throw new ArgumentException($"missing filename for {option.Key}"); } project.Output.Add(optionName, option.Value); } } return(project); }
public override bool Execute() { RemotingUtility.CleanUpRegisteredChannels(); if (TeamCity) { Log.LogError("The 'TeamCity' property is deprecated. Please set the 'Reporter' property to 'teamcity' instead."); return(false); } if (Verbose) { Log.LogError("The 'Verbose' property is deprecated. Please set the 'Reporter' property to 'verbose' instead."); return(false); } XElement assembliesElement = null; var environment = $"{IntPtr.Size * 8}-bit .NET {Environment.Version}"; if (NeedsXml) { assembliesElement = new XElement("assemblies"); } switch (MaxParallelThreads) { case null: case "default": break; case "unlimited": maxThreadCount = -1; break; default: int threadValue; if (!int.TryParse(MaxParallelThreads, out threadValue) || threadValue < 1) { Log.LogError("MaxParallelThreads value '{0}' is invalid: must be 'default', 'unlimited', or a positive number", MaxParallelThreads); return(false); } maxThreadCount = threadValue; break; } var originalWorkingFolder = Directory.GetCurrentDirectory(); using (AssemblyHelper.SubscribeResolve()) { var reporters = GetAvailableRunnerReporters(); var reporter = reporters.FirstOrDefault(r => r.IsEnvironmentallyEnabled); if (reporter == null && !string.IsNullOrWhiteSpace(Reporter)) { reporter = reporters.FirstOrDefault(r => string.Equals(r.RunnerSwitch, Reporter, StringComparison.OrdinalIgnoreCase)); if (reporter == null) { var switchableReporters = reporters.Where(r => !string.IsNullOrWhiteSpace(r.RunnerSwitch)).Select(r => r.RunnerSwitch.ToLowerInvariant()).OrderBy(x => x).ToList(); if (switchableReporters.Count == 0) { Log.LogError("Reporter value '{0}' is invalid. There are no available reporters.", Reporter); } else { Log.LogError("Reporter value '{0}' is invalid. Available reporters: {1}", Reporter, string.Join(", ", switchableReporters)); } return(false); } } if (reporter == null) { reporter = new DefaultRunnerReporter(); } logger = new MSBuildLogger(Log); reporterMessageHandler = reporter.CreateMessageHandler(logger); if (!NoLogo) { Log.LogMessage(MessageImportance.High, "xUnit.net MSBuild Runner ({0})", environment); } var project = new XunitProject(); foreach (var assembly in Assemblies) { var assemblyFileName = assembly.GetMetadata("FullPath"); var configFileName = assembly.GetMetadata("ConfigFile"); if (configFileName != null && configFileName.Length == 0) { configFileName = null; } var projectAssembly = new XunitProjectAssembly { AssemblyFilename = assemblyFileName, ConfigFilename = configFileName }; if (shadowCopy.HasValue) { projectAssembly.Configuration.ShadowCopy = shadowCopy; } project.Add(projectAssembly); } if (WorkingFolder != null) { Directory.SetCurrentDirectory(WorkingFolder); } var clockTime = Stopwatch.StartNew(); if (!parallelizeAssemblies.HasValue) { parallelizeAssemblies = project.All(assembly => assembly.Configuration.ParallelizeAssemblyOrDefault); } if (parallelizeAssemblies.GetValueOrDefault()) { var tasks = project.Assemblies.Select(assembly => Task.Run(() => ExecuteAssembly(assembly))); var results = Task.WhenAll(tasks).GetAwaiter().GetResult(); foreach (var assemblyElement in results.Where(result => result != null)) { assembliesElement.Add(assemblyElement); } } else { foreach (var assembly in project.Assemblies) { var assemblyElement = ExecuteAssembly(assembly); if (assemblyElement != null) { assembliesElement.Add(assemblyElement); } } } clockTime.Stop(); if (completionMessages.Count > 0) { reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, completionMessages.OrderBy(kvp => kvp.Key).ToList())); } } Directory.SetCurrentDirectory(WorkingFolder ?? originalWorkingFolder); if (NeedsXml) { if (Xml != null) { assembliesElement.Save(Xml.GetMetadata("FullPath")); } if (XmlV1 != null) { Transform("xUnit1.xslt", assembliesElement, XmlV1); } if (Html != null) { Transform("HTML.xslt", assembliesElement, Html); } if (NUnit != null) { Transform("NUnitXml.xslt", assembliesElement, NUnit); } } return(ExitCode == 0); }
protected XunitProject Parse(Predicate<string> fileExists) { var assemblies = new List<Tuple<string, string>>(); Reporter = new DefaultRunnerReporter(); while (arguments.Count > 0) { if (arguments.Peek().StartsWith("-", StringComparison.Ordinal)) break; var assemblyFile = arguments.Pop(); if (IsConfigFile(assemblyFile)) throw new ArgumentException($"expecting assembly, got config file: {assemblyFile}"); if (!fileExists(assemblyFile)) throw new ArgumentException($"file not found: {assemblyFile}"); string configFile = null; if (arguments.Count > 0) { var value = arguments.Peek(); if (!value.StartsWith("-", StringComparison.Ordinal) && IsConfigFile(value)) { configFile = arguments.Pop(); if (!fileExists(configFile)) throw new ArgumentException($"config file not found: {configFile}"); } } assemblies.Add(Tuple.Create(assemblyFile, configFile)); } if (assemblies.Count == 0) throw new ArgumentException("must specify at least one assembly"); var project = GetProjectFile(assemblies); while (arguments.Count > 0) { var option = PopOption(arguments); var optionName = option.Key.ToLowerInvariant(); if (!optionName.StartsWith("-", StringComparison.Ordinal)) throw new ArgumentException($"unknown command line option: {option.Key}"); optionName = optionName.Substring(1); if (optionName == "installlocation") { if (option.Value == null) throw new ArgumentException("missing argument for -installlocation"); InstallLocation = option.Value; } else if (optionName == "nologo") { GuardNoOptionValue(option); NoLogo = true; } else if (optionName == "failskips") { GuardNoOptionValue(option); FailSkips = true; } else if (optionName == "nocolor") { GuardNoOptionValue(option); NoColor = true; } else if (optionName == "noappdomain") { GuardNoOptionValue(option); NoAppDomain = true; } else if (optionName == "debug") { GuardNoOptionValue(option); Debug = true; } else if (optionName == "serialize") { GuardNoOptionValue(option); Serialize = true; } else if (optionName == "wait") { GuardNoOptionValue(option); Wait = true; } else if (optionName == "diagnostics") { GuardNoOptionValue(option); DiagnosticMessages = true; } else if (optionName == "maxthreads") { if (option.Value == null) throw new ArgumentException("missing argument for -maxthreads"); switch (option.Value) { case "default": MaxParallelThreads = 0; break; case "unlimited": MaxParallelThreads = -1; break; default: int threadValue; if (!int.TryParse(option.Value, out threadValue) || threadValue < 1) throw new ArgumentException("incorrect argument value for -maxthreads (must be 'default', 'unlimited', or a positive number)"); MaxParallelThreads = threadValue; break; } } else if (optionName == "parallel") { if (option.Value == null) throw new ArgumentException("missing argument for -parallel"); ParallelismOption parallelismOption; if (!Enum.TryParse<ParallelismOption>(option.Value, out parallelismOption)) throw new ArgumentException("incorrect argument value for -parallel"); switch (parallelismOption) { case ParallelismOption.all: ParallelizeAssemblies = true; ParallelizeTestCollections = true; break; case ParallelismOption.assemblies: ParallelizeAssemblies = true; ParallelizeTestCollections = false; break; case ParallelismOption.collections: ParallelizeAssemblies = false; ParallelizeTestCollections = true; break; default: ParallelizeAssemblies = false; ParallelizeTestCollections = false; break; } } else if (optionName == "noshadow") { GuardNoOptionValue(option); foreach (var assembly in project.Assemblies) assembly.Configuration.ShadowCopy = false; } else if (optionName == "trait") { if (option.Value == null) throw new ArgumentException("missing argument for -trait"); var pieces = option.Value.Split('='); if (pieces.Length != 2 || string.IsNullOrEmpty(pieces[0]) || string.IsNullOrEmpty(pieces[1])) throw new ArgumentException("incorrect argument format for -trait (should be \"name=value\")"); var name = pieces[0]; var value = pieces[1]; project.Filters.IncludedTraits.Add(name, value); } else if (optionName == "notrait") { if (option.Value == null) throw new ArgumentException("missing argument for -notrait"); var pieces = option.Value.Split('='); if (pieces.Length != 2 || string.IsNullOrEmpty(pieces[0]) || string.IsNullOrEmpty(pieces[1])) throw new ArgumentException("incorrect argument format for -notrait (should be \"name=value\")"); var name = pieces[0]; var value = pieces[1]; project.Filters.ExcludedTraits.Add(name, value); } else if (optionName == "class") { if (option.Value == null) throw new ArgumentException("missing argument for -class"); project.Filters.IncludedClasses.Add(option.Value); } else if (optionName == "method") { if (option.Value == null) throw new ArgumentException("missing argument for -method"); project.Filters.IncludedMethods.Add(option.Value); } else if (optionName == "namespace") { if (option.Value == null) throw new ArgumentException("missing argument for -namespace"); project.Filters.IncludedNameSpaces.Add(option.Value); } else if (optionName == "xml") { if (option.Value == null) throw new ArgumentException($"missing filename for {option.Key}"); project.Output.Add(optionName, option.Value); } } return project; }
void DiscoverTests <TVisitor>(IEnumerable <string> sources, LoggerHelper logger, Func <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitorFactory, Action <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitComplete = null) where TVisitor : IVsDiscoveryVisitor, IDisposable { try { RemotingUtility.CleanUpRegisteredChannels(); using (AssemblyHelper.SubscribeResolve()) { var reporterMessageHandler = new DefaultRunnerReporter().CreateMessageHandler(new VisualStudioRunnerLogger(logger)); foreach (var assemblyFileName in sources) { var assembly = new XunitProjectAssembly { AssemblyFilename = assemblyFileName }; var configuration = LoadConfiguration(assemblyFileName); var fileName = Path.GetFileNameWithoutExtension(assemblyFileName); var shadowCopy = configuration.ShadowCopyOrDefault; try { if (cancelled) { break; } if (!IsXunitTestAssembly(assemblyFileName)) { if (configuration.DiagnosticMessagesOrDefault) { logger.Log("Skipping: {0} (no reference to xUnit.net)", fileName); } } else { var diagnosticMessageVisitor = new DiagnosticMessageVisitor(logger, fileName, configuration.DiagnosticMessagesOrDefault); using (var framework = new XunitFrontController(AppDomain, assemblyFileName: assemblyFileName, configFileName: null, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageVisitor)) { var targetFramework = framework.TargetFramework; if (targetFramework.StartsWith("MonoTouch", StringComparison.OrdinalIgnoreCase) || targetFramework.StartsWith("MonoAndroid", StringComparison.OrdinalIgnoreCase) || targetFramework.StartsWith("Xamarin.iOS", StringComparison.OrdinalIgnoreCase)) { if (configuration.DiagnosticMessagesOrDefault) { logger.Log("Skipping: {0} (unsupported target framework '{1}')", fileName, targetFramework); } } else { var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); using (var visitor = visitorFactory(assemblyFileName, framework, discoveryOptions)) { reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, framework.CanUseAppDomains && AppDomain != AppDomainSupport.Denied, shadowCopy, discoveryOptions)); framework.Find(includeSourceInformation: true, messageSink: visitor, discoveryOptions: discoveryOptions); var totalTests = visitor.Finish(); if (visitComplete != null) { visitComplete(assemblyFileName, framework, discoveryOptions, visitor); } reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, totalTests, totalTests)); } } } } } catch (Exception e) { var ex = e.Unwrap(); var fileNotFound = ex as FileNotFoundException; #if !PLATFORM_DOTNET var fileLoad = ex as FileLoadException; #endif if (ex is InvalidOperationException) { logger.LogWarning("Skipping: {0} ({1})", fileName, ex.Message); } else if (fileNotFound != null) { logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileNotFound.FileName)); } #if !PLATFORM_DOTNET else if (fileLoad != null) { logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileLoad.FileName)); } #endif else { logger.LogWarning("Exception discovering tests from {0}: {1}", fileName, ex); } } } } } catch (Exception e) { logger.LogWarning("Exception discovering tests: {0}", e.Unwrap()); } }