private Assembly[] LoadReferencedAssemblies() { var assemblies = new List <Assembly>() { Assembly }; #if DEBUG _trace.WriteLine($"[NetCoreAssemblyRuntimeLoader][LoadReferencedAssemblies] Loading references for [{Assembly}]"); #endif foreach (var assemblyName in Assembly.GetReferencedAssemblies()) { try { #if DEBUG _trace.WriteLine($"[NetCoreAssemblyRuntimeLoader][LoadReferencedAssemblies] Attempting to load [{assemblyName}]"); #endif assemblies.Add(_loadContext.LoadFromAssemblyName(assemblyName)); } catch (Exception ex) { // exception occurred, but we don't care #if DEBUG _trace.Error(ex, $"[NetCoreAssemblyRuntimeLoader][LoadReferencedAssemblies] Failed to load [{assemblyName}]"); #endif } } return(assemblies.ToArray()); }
/// <summary> /// Executes the tests /// </summary> /// <returns>True if all tests passed.</returns> public TestRunnerResult Execute() { // Perform core / thread optimizations if we're running in single-threaded mode // But not if the user has specified that they're going to be running multi-threaded benchmarks SetProcessPriority(_package.Concurrent); IBenchmarkOutput output = CreateOutput(); var discovery = new ReflectionDiscovery(output); var result = new TestRunnerResult() { AllTestsPassed = true }; try { foreach (var testFile in _package.Files) { var assembly = AssemblyRuntimeLoader.LoadAssembly(testFile); var benchmarks = discovery.FindBenchmarks(assembly); foreach (var benchmark in benchmarks) { // verify if the benchmark should be included/excluded from the list of benchmarks to be run if (_package.ShouldRunBenchmark(benchmark.BenchmarkName)) { output.WriteLine($"------------ STARTING {benchmark.BenchmarkName} ---------- "); benchmark.Run(); benchmark.Finish(); // if one assert fails, all fail result.AllTestsPassed = result.AllTestsPassed && benchmark.AllAssertsPassed; output.WriteLine($"------------ FINISHED {benchmark.BenchmarkName} ---------- "); result.ExecutedTestsCount = result.ExecutedTestsCount + 1; } else { output.WriteLine($"------------ NOTRUN {benchmark.BenchmarkName} ---------- "); result.IgnoredTestsCount = result.IgnoredTestsCount + 1; } } } } catch (Exception ex) { output.Error(ex, "Error while executing the tests."); result.AllTestsPassed = false; } return(result); }
private static bool MethodHasValidBenchmark(MethodInfo x) { var hasPerformanceBenchmarkAttribute = x.IsDefined(PerformanceBenchmarkAttributeType, true); var hasAtLeastOneMeasurementAttribute = x.IsDefined(MeasurementAttributeType, true); var skipReason = (x.GetCustomAttribute(PerformanceBenchmarkAttributeType) as PerfBenchmarkAttribute)?.Skip; var benchmarkIsSkipped = hasPerformanceBenchmarkAttribute && !String.IsNullOrEmpty(skipReason); // code below is for adding interface support //var bla = // (from @interface in x.DeclaringType.GetInterfaces() // let map = x.DeclaringType.GetInterfaceMap(@interface) // let index = Array.IndexOf(map.TargetMethods, x) // where index >= 0 // select map.InterfaceMethods[index]).FirstOrDefault(); //var hasPerformanceBenchmarkAttributeOnInterface = bla.IsDefined(PerformanceBenchmarkAttributeType), true); /* * If user defined a PerformanceBenchmark attribute but never added on any Measurement * attributes, then we need to log a warning here. */ if (hasPerformanceBenchmarkAttribute && !hasAtLeastOneMeasurementAttribute) { ReflectionOutput.Warning($"{x.DeclaringType?.Name}+{x.Name} has a declared PerformanceBenchmarkAttribute but no declared measurements. Skipping..."); } else if (benchmarkIsSkipped) { ReflectionOutput.WriteLine($"Skipping {x.DeclaringType?.Name}+{x.Name}. Reason: {skipReason}."); } return(hasPerformanceBenchmarkAttribute && hasAtLeastOneMeasurementAttribute && !benchmarkIsSkipped); }
/// <summary> /// NBench Runner takes the following <see cref="args"/> /// /// C:\> NBench.Runner.exe [assembly name] [output-directory={dir-path}] /// /// </summary> /// <param name="args">The commandline arguments</param> static int Main(string[] args) { Output = new CompositeBenchmarkOutput(new ConsoleBenchmarkOutput(), new MarkdownBenchmarkOutput(CommandLine.GetProperty("output-directory"))); Discovery = new ReflectionDiscovery(Output); string assemblyPath = Path.GetFullPath(args[0]); // TODO: See issue https://github.com/petabridge/NBench/issues/3 var assembly = AssemblyRuntimeLoader.LoadAssembly(assemblyPath); /* * Set processor affinity */ Process Proc = Process.GetCurrentProcess(); Proc.ProcessorAffinity = new IntPtr(2); // either of the first two processors /* * Set priority */ Process.GetCurrentProcess().PriorityClass = ProcessPriorityClass.High; Thread.CurrentThread.Priority = ThreadPriority.Highest; var benchmarks = Discovery.FindBenchmarks(assembly); bool anyAssertFailures = false; foreach (var benchmark in benchmarks) { Output.WriteLine($"------------ STARTING {benchmark.BenchmarkName} ---------- "); benchmark.Run(); benchmark.Finish(); // if one assert fails, all fail anyAssertFailures = anyAssertFailures || !benchmark.AllAssertsPassed; Output.WriteLine($"------------ FINISHED {benchmark.BenchmarkName} ---------- "); } return(anyAssertFailures ? -1 : 0); }
/// <summary> /// NBench Runner takes the following <see cref="args"/> /// /// C:\> NBench.Runner.exe [assembly name] [output-directory={dir-path}] /// /// </summary> /// <param name="args">The commandline arguments</param> static int Main(string[] args) { Output = new CompositeBenchmarkOutput(new ConsoleBenchmarkOutput(), new MarkdownBenchmarkOutput(CommandLine.GetProperty("output-directory"))); Discovery = new ReflectionDiscovery(Output); string assemblyPath = Path.GetFullPath(args[0]); // TODO: See issue https://github.com/petabridge/NBench/issues/3 var assembly = AssemblyRuntimeLoader.LoadAssembly(assemblyPath); /* * Set processor affinity */ Process Proc = Process.GetCurrentProcess(); Proc.ProcessorAffinity = new IntPtr(2); // either of the first two processors /* * Set priority */ Process.GetCurrentProcess().PriorityClass = ProcessPriorityClass.High; Thread.CurrentThread.Priority = ThreadPriority.Highest; var benchmarks = Discovery.FindBenchmarks(assembly); bool anyAssertFailures = false; foreach (var benchmark in benchmarks) { Output.WriteLine($"------------ STARTING {benchmark.BenchmarkName} ---------- "); benchmark.Run(); benchmark.Finish(); // if one assert fails, all fail anyAssertFailures = anyAssertFailures || !benchmark.AllAssertsPassed; Output.WriteLine($"------------ FINISHED {benchmark.BenchmarkName} ---------- "); } return anyAssertFailures ? -1 : 0; }
private void WriteMessage(TraceMessage message) { // all traces go into the usual log. no special treatment for errors or warnings. _benchmarkOutput.WriteLine(message.ToString()); }
/// <summary> /// Executes the tests /// </summary> /// <returns>True if all tests passed.</returns> public TestRunnerResult Execute() { // Perform core / thread optimizations if we're running in single-threaded mode // But not if the user has specified that they're going to be running multi-threaded benchmarks SetProcessPriority(_package.Concurrent); // pass in the runner settings so we can include them in benchmark reports // also, toggles tracing on or off var runnerSettings = new RunnerSettings() { ConcurrentModeEnabled = _package.Concurrent, TracingEnabled = _package.Tracing }; IBenchmarkOutput output = CreateOutput(); var discovery = new ReflectionDiscovery(output, DefaultBenchmarkAssertionRunner.Instance, // one day we might be able to pass in custom assertion runners, hence why this is here runnerSettings); var result = new TestRunnerResult() { AllTestsPassed = true }; try { foreach (var testFile in _package.Files) { output.WriteLine($"Executing Benchmarks in {testFile}"); using (var assembly = AssemblyRuntimeLoader.LoadAssembly(testFile, output)) { var benchmarks = discovery.FindBenchmarks(assembly.Assembly); foreach (var benchmark in benchmarks) { // verify if the benchmark should be included/excluded from the list of benchmarks to be run if (_package.ShouldRunBenchmark(benchmark.BenchmarkName)) { output.StartBenchmark(benchmark.BenchmarkName); benchmark.Run(); benchmark.Finish(); // if one assert fails, all fail result.AllTestsPassed = result.AllTestsPassed && benchmark.AllAssertsPassed; output.FinishBenchmark(benchmark.BenchmarkName); result.ExecutedTestsCount = result.ExecutedTestsCount + 1; } else { output.SkipBenchmark(benchmark.BenchmarkName); result.IgnoredTestsCount = result.IgnoredTestsCount + 1; } } } } } catch (Exception ex) { output.Error(ex, "Error while executing the tests."); result.AllTestsPassed = false; } return(result); }