private void RunCompetitions(string[] args) { for (int i = 0; i < Competitions.Length; i++) { var competition = Competitions[i]; if (args.Any(arg => competition.Name.ToLower().StartsWith(arg.ToLower())) || args.Contains("#" + i) || args.Contains("*")) { logger.WriteLineHeader("Target competition: " + competition.Name); List<BenchmarkReport> reports; using (var logStreamWriter = new StreamWriter(competition.Name + ".log")) { var loggers = new IBenchmarkLogger[] { new BenchmarkConsoleLogger(), new BenchmarkStreamLogger(logStreamWriter) }; var runner = new BenchmarkRunner(loggers); reports = runner.RunCompetition(Activator.CreateInstance(competition), BenchmarkSettings.Parse(args)).ToList(); } MarkdownReportExporter.Default.SaveToFile(reports, competition.Name + "-report.md"); CsvReportExporter.Default.SaveToFile(reports, competition.Name + "-report.csv"); logger.NewLine(); } } if (args.Length > 0 && (args[0].StartsWith("http://") || args[0].StartsWith("https://"))) { var url = args[0]; Uri uri = new Uri(url); var name = uri.IsFile ? Path.GetFileName(uri.LocalPath) : "URL"; using (var logStreamWriter = new StreamWriter(name + ".log")) { var loggers = new IBenchmarkLogger[] { new BenchmarkConsoleLogger(), new BenchmarkStreamLogger(logStreamWriter) }; var runner = new BenchmarkRunner(loggers); runner.RunUrl(url, BenchmarkSettings.Parse(args)); } } }
public override void Export(IList<BenchmarkReport> reports, IBenchmarkLogger logger, IEnumerable<IBenchmarkResultExtender> resultExtenders = null) { if(useCodeBlocks) logger.WriteLine($"```{codeBlocksSyntax}"); logger = new BenchmarkLoggerWithPrefix(logger, prefix); logger.WriteLineInfo(EnvironmentInfo.GetCurrentInfo().ToFormattedString("Host")); logger.NewLine(); var table = BenchmarkExporterHelper.BuildTable(reports, resultExtenders); // If we have Benchmarks with ParametersSets, force the "Method" columns to be displayed, otherwise it doesn't make as much sense var columnsToAlwaysShow = (reports.Any(r => r.Benchmark.Task.ParametersSets != null) ? new[] { "Method" } : new string[0]). Concat((resultExtenders ?? new IBenchmarkResultExtender[0]).Select(e => e.ColumnName)).ToArray(); PrintTable(table, logger, columnsToAlwaysShow); // TODO: move this logic to an analyser var benchmarksWithTroubles = reports.Where(r => !r.GetTargetRuns().Any()).Select(r => r.Benchmark).ToList(); if (benchmarksWithTroubles.Count > 0) { logger.NewLine(); logger.WriteLineError("Benchmarks with troubles:"); foreach (var benchmarkWithTroubles in benchmarksWithTroubles) logger.WriteLineError(" " + benchmarkWithTroubles.Caption); } }
private void PrintTable(List<string[]> table, IBenchmarkLogger logger) { int rowCount = table.Count, colCount = table[0].Length; int[] widths = new int[colCount]; bool[] areSame = new bool[colCount]; for (int colIndex = 0; colIndex < colCount; colIndex++) { areSame[colIndex] = rowCount > 2 && colIndex < colCount - 3; for (int rowIndex = 0; rowIndex < rowCount; rowIndex++) { widths[colIndex] = Math.Max(widths[colIndex], table[rowIndex][colIndex].Length + 1); if (rowIndex > 1 && table[rowIndex][colIndex] != table[1][colIndex]) areSame[colIndex] = false; } } if (areSame.Any(s => s)) { for (int colIndex = 0; colIndex < colCount; colIndex++) if (areSame[colIndex]) logger.WriteInfo($"{table[0][colIndex]}={table[1][colIndex]} "); logger.NewLine(); logger.WriteLineInfo("```"); logger.NewLine(); } table.Insert(1, widths.Select(w => new string('-', w)).ToArray()); foreach (var row in table) { for (int colIndex = 0; colIndex < colCount; colIndex++) if (!areSame[colIndex]) logger.WriteStatistic(row[colIndex].PadLeft(widths[colIndex], ' ') + " |"); logger.NewLine(); } }
public void Export(IList <BenchmarkReport> reports, IBenchmarkLogger logger) { foreach (var exporter in exporters) { exporter.Export(reports, logger); } }
public BenchmarkClassicFlow(Benchmark benchmark, IBenchmarkLogger logger) { this.benchmark = benchmark; generator = new BenchmarkClassicGenerator(logger); builder = new BenchmarkClassicBuilder(logger); executor = new BenchmarkClassicExecutor(benchmark, logger); }
public override void Export(IList<BenchmarkReport> reports, IBenchmarkLogger logger, IEnumerable<IBenchmarkResultExtender> resultExtenders = null) { logger.WriteLine(string.Join(";", columns.Select(c => c.Title))); foreach (var report in reports) foreach (var run in report.Runs) logger.WriteLine(string.Join(";", columns.Select(column => column.GetValue(report, run)))); }
private void PrintTable(List<string[]> table, IBenchmarkLogger logger, string[] columnsToAlwaysShow) { if (table.Count == 0) { logger.WriteLineError("There are no found benchmarks"); logger.NewLine(); return; } int rowCount = table.Count, colCount = table[0].Length; var columnsToShowIndexes = columnsToAlwaysShow.Select(col => Array.IndexOf(table[0], col)).ToArray(); int[] widths = new int[colCount]; bool[] areSame = new bool[colCount]; for (int colIndex = 0; colIndex < colCount; colIndex++) { areSame[colIndex] = rowCount > 2 && colIndex < colCount; for (int rowIndex = 0; rowIndex < rowCount; rowIndex++) { widths[colIndex] = Math.Max(widths[colIndex], table[rowIndex][colIndex].Length + 1); if (rowIndex > 1 && table[rowIndex][colIndex] != table[1][colIndex]) areSame[colIndex] = false; } } if (areSame.Any(s => s)) { var paramsOnLine = 0; for (int colIndex = 0; colIndex < colCount; colIndex++) if (areSame[colIndex] && columnsToShowIndexes.Contains(colIndex) == false) { logger.WriteInfo($"{table[0][colIndex]}={table[1][colIndex]} "); paramsOnLine++; if (paramsOnLine == 3) { logger.NewLine(); paramsOnLine = 0; } } logger.NewLine(); } if (useCodeBlocks) { logger.Write("```"); logger.NewLine(); } table.Insert(1, widths.Select(w => new string('-', w)).ToArray()); foreach (var row in table) { for (int colIndex = 0; colIndex < colCount; colIndex++) if (!areSame[colIndex] || columnsToShowIndexes.Contains(colIndex)) logger.WriteStatistic(row[colIndex].PadLeft(widths[colIndex], ' ') + " |"); logger.NewLine(); } if (areSame.Any(s => s)) { logger.NewLine(); } }
/// <summary> /// Parses the benchmark statistics from the plain text line. /// /// E.g. given the input <paramref name="line"/>: /// /// Target 1: 10 op, 1005.8 ms, 1005842518 ns, 3332139 ticks, 100584251.7955 ns/op, 9.9 op/s /// /// Will extract the number of <see cref="Operations"/> performed and the /// total number of <see cref="Nanoseconds"/> it took to perform them. /// </summary> /// <param name="logger">The logger to write any diagnostic messages to.</param> /// <param name="line">The line to parse.</param> /// <returns>An instance of <see cref="BenchmarkRunReport"/> if parsed successfully. <c>Null</c> in case of any trouble.</returns> public static BenchmarkRunReport Parse(IBenchmarkLogger logger, string line) { try { var op = 1L; var ns = double.PositiveInfinity; var items = line. Split(new[] {':'}, StringSplitOptions.RemoveEmptyEntries)[1]. Split(new[] {','}, StringSplitOptions.RemoveEmptyEntries); foreach (var item in items) { var split = item.Split(new[] {' '}, StringSplitOptions.RemoveEmptyEntries); var unit = split[1]; switch (unit) { case "ns": ns = double.Parse(split[0], EnvironmentHelper.MainCultureInfo); break; case "op": op = long.Parse(split[0]); break; } } return new BenchmarkRunReport(op, ns); } catch (Exception) { logger.WriteLineError("Parse error in the following line:"); logger.WriteLineError(line); return null; } }
private BenchmarkReport Run(IBenchmarkLogger logger, Benchmark benchmark, IList <string> importantPropertyNames, BenchmarkParameters parameters = null) { var toolchain = Plugins.CreateToolchain(benchmark, logger); logger.WriteLineHeader("// **************************"); logger.WriteLineHeader("// Benchmark: " + benchmark.Description); var generateResult = Generate(logger, toolchain); if (!generateResult.IsGenerateSuccess) { return(BenchmarkReport.CreateEmpty(benchmark, parameters)); } var buildResult = Build(logger, toolchain, generateResult); if (!buildResult.IsBuildSuccess) { return(BenchmarkReport.CreateEmpty(benchmark, parameters)); } var runReports = Execute(logger, benchmark, importantPropertyNames, parameters, toolchain, buildResult); return(new BenchmarkReport(benchmark, runReports, parameters)); }
/// <summary> /// Parses the benchmark statistics from the plain text line. /// /// E.g. given the input <paramref name="line"/>: /// /// Target 1: 10 op, 1005.8 ms, 1005842518 ns, 3332139 ticks, 100584251.7955 ns/op, 9.9 op/s /// /// Will extract the number of <see cref="Operations"/> performed and the /// total number of <see cref="Nanoseconds"/> it took to perform them. /// </summary> /// <param name="logger">The logger to write any diagnostic messages to.</param> /// <param name="line">The line to parse.</param> /// <returns>An instance of <see cref="BenchmarkRunReport"/> if parsed successfully. <c>Null</c> in case of any trouble.</returns> public static BenchmarkRunReport Parse(IBenchmarkLogger logger, string line) { try { var op = 1L; var ns = double.PositiveInfinity; var items = line. Split(new[] { ':' }, StringSplitOptions.RemoveEmptyEntries)[1]. Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries); foreach (var item in items) { var split = item.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); var unit = split[1]; switch (unit) { case "ns": ns = double.Parse(split[0], EnvironmentHelper.MainCultureInfo); break; case "op": op = long.Parse(split[0]); break; } } return(new BenchmarkRunReport(op, ns)); } catch (Exception) { logger.WriteLineError("Parse error in the following line:"); logger.WriteLineError(line); return(null); } }
public void Print(Benchmark benchmark, Process process, IBenchmarkLogger logger) { foreach (var diagnoster in diagnosers) { diagnoster.Print(benchmark, process, logger); } }
private void RunCompetitions(string[] args) { for (int i = 0; i < Competitions.Length; i++) { var competition = Competitions[i]; if (args.Any(arg => competition.Name.ToLower().StartsWith(arg.ToLower())) || args.Contains("#" + i) || args.Contains("*")) { logger.WriteLineHeader("Target competition: " + competition.Name); using (var logStreamWriter = new StreamWriter(competition.Name + ".log")) { var loggers = new IBenchmarkLogger[] { new BenchmarkConsoleLogger(), new BenchmarkStreamLogger(logStreamWriter) }; var runner = new BenchmarkRunner(loggers); runner.RunCompetition(Activator.CreateInstance(competition), BenchmarkSettings.Parse(args)); } logger.NewLine(); } } if (args.Length > 0 && (args[0].StartsWith("http://") || args[0].StartsWith("https://"))) { var url = args[0]; Uri uri = new Uri(url); var name = uri.IsFile ? Path.GetFileName(uri.LocalPath) : "URL"; using (var logStreamWriter = new StreamWriter(name + ".log")) { var loggers = new IBenchmarkLogger[] { new BenchmarkConsoleLogger(), new BenchmarkStreamLogger(logStreamWriter) }; var runner = new BenchmarkRunner(loggers); runner.RunUrl(url, BenchmarkSettings.Parse(args)); } } }
private List<BenchmarkReport> Run(List<Benchmark> benchmarks, IBenchmarkLogger logger) { logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****"); logger.WriteLineInfo("// Found benchmarks:"); foreach (var benchmark in benchmarks) logger.WriteLineInfo($"// {benchmark.Description}"); logger.NewLine(); var importantPropertyNames = benchmarks.Select(b => b.Properties).GetImportantNames(); var reports = new List<BenchmarkReport>(); foreach (var benchmark in benchmarks) { if (benchmark.Task.ParametersSets.IsEmpty()) { var report = Run(logger, benchmark, importantPropertyNames); reports.Add(report); if (report.Runs.Count > 0) { var stat = new BenchmarkRunReportsStatistic("Target", report.Runs); logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}"); logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}"); } } else { var parametersSets = benchmark.Task.ParametersSets; foreach (var parameters in parametersSets.ToParameters()) { var report = Run(logger, benchmark, importantPropertyNames, parameters); reports.Add(report); if (report.Runs.Count > 0) { var stat = new BenchmarkRunReportsStatistic("Target", report.Runs); logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}"); logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}"); } } } logger.NewLine(); } logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****"); logger.NewLine(); BenchmarkMarkdownExporter.Default.Export(reports, logger); var warnings = Plugins.CompositeAnalyser.Analyze(reports).ToList(); if (warnings.Count > 0) { logger.NewLine(); logger.WriteLineError("// *** Warnings *** "); foreach (var warning in warnings) logger.WriteLineError($"{warning.Message}"); } logger.NewLine(); logger.WriteLineHeader("// ***** BenchmarkRunner: End *****"); return reports; }
public static IBenchmarkToolchainFacade CreateToolchain(Benchmark benchmark, IBenchmarkLogger logger) { switch (benchmark.Task.Configuration.Toolchain) { case BenchmarkToolchain.Classic: return new BenchmarkToolchainFacade(benchmark, new BenchmarkClassicGenerator(logger), new BenchmarkClassicBuilder(logger), new BenchmarkClassicExecutor(benchmark, logger)); default: throw new NotSupportedException(); } }
public static IBenchmarkFlow CreateFlow(Benchmark benchmark, IBenchmarkLogger logger) { switch (benchmark.Task.Configuration.Executor) { case BenchmarkExecutor.Classic: return new BenchmarkClassicFlow(benchmark, logger); default: throw new NotSupportedException(); } }
public BuildResult BuildProject(string directoryPath, IBenchmarkLogger logger) { string projectFileName = Path.Combine(directoryPath, MainClassName + ".csproj"); var consoleLogger = new MSBuildConsoleLogger(logger); var globalProperties = new Dictionary<string, string>(); var buildRequest = new BuildRequestData(projectFileName, globalProperties, null, new[] { "Build" }, null); var buildParameters = new BuildParameters(new ProjectCollection()) { DetailedSummary = false, Loggers = new ILogger[] { consoleLogger } }; var buildResult = BuildManager.DefaultBuildManager.Build(buildParameters, buildRequest); return buildResult; }
private void PrintTable(List <string[]> table, IBenchmarkLogger logger, string[] columnsToAlwaysShow) { if (table.Count == 0) { logger.WriteLineError("There are no found benchmarks"); logger.NewLine(); return; } int rowCount = table.Count, colCount = table[0].Length; var columnsToShowIndexes = columnsToAlwaysShow.Select(col => Array.IndexOf(table[0], col)); int[] widths = new int[colCount]; bool[] areSame = new bool[colCount]; for (int colIndex = 0; colIndex < colCount; colIndex++) { areSame[colIndex] = rowCount > 2 && colIndex < colCount - 3; for (int rowIndex = 0; rowIndex < rowCount; rowIndex++) { widths[colIndex] = Math.Max(widths[colIndex], table[rowIndex][colIndex].Length + 1); if (rowIndex > 1 && table[rowIndex][colIndex] != table[1][colIndex]) { areSame[colIndex] = false; } } } if (areSame.Any(s => s)) { for (int colIndex = 0; colIndex < colCount; colIndex++) { if (areSame[colIndex] && columnsToShowIndexes.Contains(colIndex) == false) { logger.WriteInfo($"{table[0][colIndex]}={table[1][colIndex]} "); } } logger.NewLine(); } table.Insert(1, widths.Select(w => new string('-', w)).ToArray()); foreach (var row in table) { for (int colIndex = 0; colIndex < colCount; colIndex++) { if (!areSame[colIndex] || columnsToShowIndexes.Contains(colIndex)) { logger.WriteStatistic(row[colIndex].PadLeft(widths[colIndex], ' ') + " |"); } } logger.NewLine(); } if (areSame.Any(s => s)) { logger.NewLine(); } }
public BenchmarkClassicExecutor(Benchmark benchmark, IBenchmarkLogger logger) { this.benchmark = benchmark; this.logger = logger; if (consoleHandler == null) { consoleHandler = new ConsoleHandler(logger); Console.CancelKeyPress += consoleHandler.EventHandler; } }
public IBenchmarkToolchainFacade CreateToolchain(Benchmark benchmark, IBenchmarkLogger logger) { var toolchain = benchmark.Task.Configuration.Toolchain; var targetToolchainBuilder = toolchains.FirstOrDefault(t => t.TargetToolchain == toolchain); if (targetToolchainBuilder != null) { return(targetToolchainBuilder.Build(benchmark, logger)); } throw new NotSupportedException($"There are no toolchain implementations for the '{toolchain}' toolchain"); }
public BenchmarkCodeExtractor(Benchmark benchmark, Process process, string codeExeName, IBenchmarkLogger logger) { this.process = process; this.codeExeName = codeExeName; this.logger = logger; //Method name format: "BenchmarkDotNet.Samples.Infra.RunFast()" (NOTE: WITHOUT the return type) var methodInfo = benchmark.Target.Method; fullTypeName = methodInfo.DeclaringType.FullName; var methodParams = string.Join(", ", methodInfo.GetParameters().Select(p => p.ParameterType.FullName)); fullMethodName = $"{fullTypeName}.{methodInfo.Name}({methodParams})"; }
public override void Export(IList<BenchmarkReport> reports, IBenchmarkLogger logger, IEnumerable<IBenchmarkResultExtender> resultExtenders = null) { var table = BenchmarkExporterHelper.BuildTable(reports, resultExtenders, false); foreach (var line in table) { for (int i = 0; i < line.Length; i++) { if (i != 0) logger.Write(";"); logger.Write(line[i]); } logger.NewLine(); } }
public void Export(IList<BenchmarkReport> reports, IBenchmarkLogger logger) { var table = ReportExporterHelper.BuildTable(reports, false); foreach (var line in table) { for (int i = 0; i < line.Length; i++) { if (i != 0) logger.Write(";"); logger.Write(line[i]); } logger.NewLine(); } }
public void Export(IList<BenchmarkReport> reports, IBenchmarkLogger logger) { logger.WriteLineInfo("```ini"); logger.WriteLineInfo(EnvironmentHelper.GetFullEnvironmentInfo("Host", false)); var table = ReportExporterHelper.BuildTable(reports); PrintTable(table, logger); var benchmarksWithTroubles = reports.Where(r => r.Runs.Count == 0).Select(r => r.Benchmark).ToList(); if (benchmarksWithTroubles.Count > 0) { logger.NewLine(); logger.WriteLineError("Benchmarks with troubles:"); foreach (var benchmarkWithTroubles in benchmarksWithTroubles) logger.WriteLineError(" " + benchmarkWithTroubles.Caption); } }
public void Export(IList <BenchmarkReport> reports, IBenchmarkLogger logger) { var table = BenchmarkExporterHelper.BuildTable(reports, false, true); foreach (var line in table) { for (int i = 0; i < line.Length; i++) { if (i != 0) { logger.Write(";"); } logger.Write(line[i]); } logger.NewLine(); } }
public void Export(IList<BenchmarkReport> reports, IBenchmarkLogger logger) { logger.WriteLineInfo(EnvironmentHelper.GetFullEnvironmentInfo("Host", false)); var table = BenchmarkExporterHelper.BuildTable(reports); // If we have Benchmarks with ParametersSets, force the "Method" columns to be displayed, otherwise it doesn't make as much sense var columnsToAlwaysShow = reports.Any(r => r.Benchmark.Task.ParametersSets != null) ? new[] { "Method" } : new string[0]; PrintTable(table, logger, columnsToAlwaysShow); var benchmarksWithTroubles = reports.Where(r => r.Runs.Count == 0).Select(r => r.Benchmark).ToList(); if (benchmarksWithTroubles.Count > 0) { logger.NewLine(); logger.WriteLineError("Benchmarks with troubles:"); foreach (var benchmarkWithTroubles in benchmarksWithTroubles) logger.WriteLineError(" " + benchmarkWithTroubles.Caption); } }
private void RunCompetitions(string[] args) { for (int i = 0; i < Competitions.Length; i++) { var competition = Competitions[i]; if (args.Any(arg => competition.Name.ToLower().StartsWith(arg.ToLower())) || args.Contains("#" + i) || args.Contains("*")) { logger.WriteLineHeader("Target competition: " + competition.Name); using (var logStreamWriter = new StreamWriter(competition.Name + ".log")) { var loggers = new IBenchmarkLogger[] {new BenchmarkConsoleLogger(), new BenchmarkStreamLogger(logStreamWriter)}; var runner = new BenchmarkRunner(loggers); runner.RunCompetition(Activator.CreateInstance(competition), BenchmarkSettings.Parse(args)); } logger.NewLine(); } } }
public override void Export(IList<BenchmarkReport> reports, IBenchmarkLogger logger, IEnumerable<IBenchmarkResultExtender> resultExtenders = null) { foreach (var report in reports) { var runs = report.Runs; var modes = runs.Select(it => it.IterationMode).Distinct(); logger.WriteLineHeader($"*** {report.Benchmark.Description} ***"); logger.WriteLineHeader("* Raw *"); foreach (var run in runs) logger.WriteLineResult(run.ToStr()); foreach (var mode in modes) { logger.NewLine(); logger.WriteLineHeader($"* Statistics for {mode}"); logger.WriteLineStatistic(runs.Where(it => it.IterationMode == mode).GetStats().ToTimeStr()); } } }
private BenchmarkBuildResult Build(IBenchmarkLogger logger, IBenchmarkToolchainFacade toolchain, BenchmarkGenerateResult generateResult) { logger.WriteLineInfo("// *** Build ***"); var buildResult = toolchain.Build(generateResult); if (buildResult.IsBuildSuccess) { logger.WriteLineInfo("// Result = Success"); } else { logger.WriteLineError("// Result = Failure"); if (buildResult.BuildException != null) { logger.WriteLineError($"// Exception: {buildResult.BuildException.Message}"); } } logger.NewLine(); return(buildResult); }
private BenchmarkGenerateResult Generate(IBenchmarkLogger logger, IBenchmarkToolchainFacade toolchain) { logger.WriteLineInfo("// *** Generate *** "); var generateResult = toolchain.Generate(); if (generateResult.IsGenerateSuccess) { logger.WriteLineInfo("// Result = Success"); logger.WriteLineInfo($"// {nameof(generateResult.DirectoryPath)} = {generateResult.DirectoryPath}"); } else { logger.WriteLineError("// Result = Failure"); if (generateResult.GenerateException != null) { logger.WriteLineError($"// Exception: {generateResult.GenerateException.Message}"); } } logger.NewLine(); return(generateResult); }
public void Export(IList <BenchmarkReport> reports, IBenchmarkLogger logger) { logger.WriteLineInfo(EnvironmentHelper.GetFullEnvironmentInfo("Host", false)); var table = BenchmarkExporterHelper.BuildTable(reports); // If we have Benchmarks with ParametersSets, force the "Method" columns to be displayed, otherwise it doesn't make as much sense var columnsToAlwaysShow = reports.Any(r => r.Benchmark.Task.ParametersSets != null) ? new[] { "Method" } : new string[0]; PrintTable(table, logger, columnsToAlwaysShow); var benchmarksWithTroubles = reports.Where(r => r.Runs.Count == 0).Select(r => r.Benchmark).ToList(); if (benchmarksWithTroubles.Count > 0) { logger.NewLine(); logger.WriteLineError("Benchmarks with troubles:"); foreach (var benchmarkWithTroubles in benchmarksWithTroubles) { logger.WriteLineError(" " + benchmarkWithTroubles.Caption); } } }
/// <summary> /// Parses the benchmark statistics from the plain text line. /// /// E.g. given the input <paramref name="line"/>: /// /// Target 1: 10 op, 1005842518 ns /// /// Will extract the number of <see cref="Operations"/> performed and the /// total number of <see cref="Nanoseconds"/> it took to perform them. /// </summary> /// <param name="logger">The logger to write any diagnostic messages to.</param> /// <param name="line">The line to parse.</param> /// <returns>An instance of <see cref="BenchmarkRunReport"/> if parsed successfully. <c>Null</c> in case of any trouble.</returns> public static BenchmarkRunReport Parse(IBenchmarkLogger logger, string line) { try { var lineSplit = line.Split(new[] { ':' }, StringSplitOptions.RemoveEmptyEntries); var iterationInfo = lineSplit[0]; var iterationInfoSplit = iterationInfo.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); var iterationMode = ParseIterationMode(iterationInfoSplit[0]); var iterationIndex = 0; int.TryParse(iterationInfoSplit[1], out iterationIndex); var measurementsInfo = lineSplit[1]; var measurementsInfoSplit = measurementsInfo.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries); var op = 1L; var ns = double.PositiveInfinity; foreach (var item in measurementsInfoSplit) { var measurementSplit = item.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); var value = measurementSplit[0]; var unit = measurementSplit[1]; switch (unit) { case "ns": ns = double.Parse(value, EnvironmentInfo.MainCultureInfo); break; case "op": op = long.Parse(value); break; } } return new BenchmarkRunReport(iterationMode, iterationIndex, op, ns); } catch (Exception) { logger.WriteLineError("Parse error in the following line:"); logger.WriteLineError(line); return null; } }
private List <BenchmarkRunReport> Execute(IBenchmarkLogger logger, Benchmark benchmark, IList <string> importantPropertyNames, BenchmarkParameters parameters, IBenchmarkToolchainFacade toolchain, BenchmarkBuildResult buildResult) { logger.WriteLineInfo("// *** Execute ***"); var processCount = Math.Max(1, benchmark.Task.ProcessCount); var runReports = new List <BenchmarkRunReport>(); for (int processNumber = 0; processNumber < processCount; processNumber++) { logger.WriteLineInfo($"// Run, Process: {processNumber + 1} / {processCount}"); if (parameters != null) { logger.WriteLineInfo($"// {parameters.ToInfo()}"); } if (importantPropertyNames.Any()) { logger.WriteInfo("// "); foreach (var name in importantPropertyNames) { logger.WriteInfo($"{name}={benchmark.Properties.GetValue(name)} "); } logger.NewLine(); } var execResult = toolchain.Execute(buildResult, parameters, Plugins.CompositeDiagnoser); if (execResult.FoundExecutable) { var iterRunReports = execResult.Data.Select(line => BenchmarkRunReport.Parse(logger, line)).Where(r => r != null).ToList(); runReports.AddRange(iterRunReports); } else { logger.WriteLineError("Executable not found"); } } logger.NewLine(); return(runReports); }
public IBenchmarkToolchainFacade Build(Benchmark benchmark, IBenchmarkLogger logger) { return(new BenchmarkToolchainFacade(benchmark, createGenerator(benchmark, logger), createBuilder(benchmark, logger), createExecutor(benchmark, logger))); }
public static void WriteLine(this IBenchmarkLogger logger) { logger.WriteLine(BenchmarkLogKind.Default, ""); }
public void Print(Benchmark benchmark, Process process, IBenchmarkLogger logger) { foreach (var diagnoster in diagnosers) diagnoster.Print(benchmark, process, logger); }
public IBenchmarkToolchainFacade Build(Benchmark benchmark, IBenchmarkLogger logger) { return new BenchmarkToolchainFacade(benchmark, createGenerator(benchmark, logger), createBuilder(benchmark, logger), createExecutor(benchmark, logger)); }
public void Export(IList<BenchmarkReport> reports, IBenchmarkLogger logger, IEnumerable<IBenchmarkResultExtender> resultExtenders = null) { ExportResultExtenders = resultExtenders; ExportCalled = true; }
public ConsoleHandler( IBenchmarkLogger logger) { this.logger = logger; EventHandler = new ConsoleCancelEventHandler(HandlerCallback); }
public BenchmarkClassicBuilder(IBenchmarkLogger logger) { this.logger = logger; }
public static void NewLine(this IBenchmarkLogger logger) { logger.Write(BenchmarkLogKind.Default, Environment.NewLine); }
public static void WriteError(this IBenchmarkLogger logger, string format, params object[] args) { logger.Write(BenchmarkLogKind.Error, format, args); }
public static void WriteLine(this IBenchmarkLogger logger, BenchmarkLogKind logKind, string format, params object[] args) { logger.Write(logKind, (format ?? string.Empty) + Environment.NewLine, args); }
public BenchmarkClassicExecutor(Benchmark benchmark, IBenchmarkLogger logger) { this.benchmark = benchmark; this.logger = logger; }
public BenchmarkExecutor(IBenchmarkLogger logger = null, bool monoMode = false) { Logger = logger; MonoMode = monoMode; }
public IBenchmarkPluginBuilder AddLogger(IBenchmarkLogger logger) { loggers.Add(logger); return(this); }
private List <BenchmarkReport> Run(List <Benchmark> benchmarks, IBenchmarkLogger logger) { logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****"); logger.WriteLineInfo("// Found benchmarks:"); foreach (var benchmark in benchmarks) { logger.WriteLineInfo($"// {benchmark.Description}"); } logger.NewLine(); var importantPropertyNames = benchmarks.Select(b => b.Properties).GetImportantNames(); var reports = new List <BenchmarkReport>(); foreach (var benchmark in benchmarks) { if (benchmark.Task.ParametersSets.IsEmpty()) { var report = Run(logger, benchmark, importantPropertyNames); reports.Add(report); if (report.Runs.Count > 0) { var stat = new BenchmarkRunReportsStatistic("Target", report.Runs); logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}"); logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}"); } } else { var parametersSets = benchmark.Task.ParametersSets; foreach (var parameters in parametersSets.ToParameters()) { var report = Run(logger, benchmark, importantPropertyNames, parameters); reports.Add(report); if (report.Runs.Count > 0) { var stat = new BenchmarkRunReportsStatistic("Target", report.Runs); logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}"); logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}"); } } } logger.NewLine(); } logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****"); logger.NewLine(); BenchmarkMarkdownExporter.Default.Export(reports, logger); var warnings = Plugins.CompositeAnalyser.Analyze(reports).ToList(); if (warnings.Count > 0) { logger.NewLine(); logger.WriteLineError("// *** Warnings *** "); foreach (var warning in warnings) { logger.WriteLineError($"{warning.Message}"); } } logger.NewLine(); logger.WriteLineHeader("// ***** BenchmarkRunner: End *****"); return(reports); }
public void Print(Benchmark benchmark, Process process, IBenchmarkLogger logger) { PrintCodeForMethod(benchmark, process, logger, true, true, false); }
public void Export(IList<BenchmarkReport> reports, IBenchmarkLogger logger) { foreach (var exporter in exporters) exporter.Export(reports, logger); }
/// <summary> /// Code from http://stackoverflow.com/questions/2057781/is-there-a-way-to-get-the-stacktraces-for-all-threads-in-c-like-java-lang-thre/24315960#24315960 /// also see http://stackoverflow.com/questions/31633541/clrmd-throws-exception-when-creating-runtime/31745689#31745689 /// </summary> public void PrintCodeForMethod(Benchmark benchmark, Process process, IBenchmarkLogger logger, bool printAssembly, bool printIL, bool printDiagnostics) { this.process = process; this.logger = logger; //Method name format: "BenchmarkDotNet.Samples.Infra.RunFast()" (NOTE: WITHOUT the return type) var methodInfo = benchmark.Target.Method; fullTypeName = methodInfo.DeclaringType.FullName; var methodParams = string.Join(", ", methodInfo.GetParameters().Select(p => p.ParameterType.FullName)); fullMethodName = $"{fullTypeName}.{methodInfo.Name}({methodParams})"; logger?.WriteLine($"\nPrinting Code for Method: {fullMethodName}"); logger?.WriteLine($"\nPrintAssembly={printAssembly}, PrintIL={printIL}"); logger?.WriteLine($"Attaching to process {Path.GetFileName(process.MainModule.FileName)}, Pid={process.Id}"); logger?.WriteLine($"Path {process.MainModule.FileName}"); using (var dataTarget = DataTarget.AttachToProcess(process.Id, 5000, AttachFlag.NonInvasive)) { var runtime = SetupClrRuntime(dataTarget); if (printDiagnostics) { PrintRuntimeDiagnosticInfo(dataTarget, runtime); } if (printAssembly == false && printIL == false) { return; } ClrType @class = runtime.GetHeap().GetTypeByName(fullTypeName); ClrMethod @method = @class.Methods.Single(m => m.GetFullSignature() == fullMethodName); DesktopModule module = (DesktopModule)@method.Type.Module; if (!module.IsPdbLoaded) { string pdbLocation = module.TryDownloadPdb(null); if (pdbLocation != null) { module.LoadPdb(pdbLocation); } } logger?.WriteLine($"Module: {Path.GetFileName(module.Name)}"); logger?.WriteLine($"Type: {method.Type.Name}"); logger?.WriteLine($"Method: {method.Name}"); // TODO work out why this returns locations inside OTHER methods, it's like it doesn't have an upper bound and just keeps going!? var ilOffsetLocations = module.GetSourceLocationsForMethod(@method.MetadataToken); string filePath = null; string[] lines = null; logger?.WriteLine(""); for (int i = 0; i < ilOffsetLocations.Count; i++) { var location = ilOffsetLocations[i]; var ilMaps = @method.ILOffsetMap.Where(il => il.ILOffset == location.ILOffset).ToList(); if (ilMaps.Any() == false) { continue; } if (lines == null || location.SourceLocation.FilePath != filePath) { filePath = location.SourceLocation.FilePath; lines = File.ReadAllLines(filePath); logger?.WriteLine($"Parsing file {Path.GetFileName(location.SourceLocation.FilePath)}"); } PrintLocationAndILMapInfo(@method, location, ilMaps); PrintSourceCode(lines, location); if (printAssembly) { var debugControl = dataTarget.DebuggerInterface as IDebugControl; PrintAssemblyCode(@method, ilMaps, runtime, debugControl); } } } }
public BenchmarkClassicGenerator(IBenchmarkLogger logger) { this.logger = logger; }
public MSBuildConsoleLogger(IBenchmarkLogger logger) { Logger = logger; }
public static void WriteLineInfo(this IBenchmarkLogger logger, string format, params object[] args) { logger.WriteLine(BenchmarkLogKind.Info, format, args); }
private List<BenchmarkReport> Run(List<Benchmark> benchmarks, IBenchmarkLogger logger, string competitionName) { logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****"); logger.WriteLineInfo("// Found benchmarks:"); foreach (var benchmark in benchmarks) logger.WriteLineInfo($"// {benchmark.Description}"); logger.NewLine(); var importantPropertyNames = benchmarks.Select(b => b.Properties).GetImportantNames(); var globalStopwatch = Stopwatch.StartNew(); var reports = new List<BenchmarkReport>(); foreach (var benchmark in benchmarks) { if (benchmark.Task.ParametersSets.IsEmpty()) { var report = Run(logger, benchmark, importantPropertyNames); reports.Add(report); if (report.GetTargetRuns().Any()) logger.WriteLineStatistic(report.GetTargetRuns().GetStats().ToTimeStr()); } else { var parametersSets = benchmark.Task.ParametersSets; foreach (var parameters in parametersSets.ToParameters()) { var report = Run(logger, benchmark, importantPropertyNames, parameters); reports.Add(report); if (report.GetTargetRuns().Any()) logger.WriteLineStatistic(report.GetTargetRuns().GetStats().ToTimeStr()); } } logger.NewLine(); } globalStopwatch.Stop(); logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****"); logger.NewLine(); logger.WriteLineHeader("// * Export *"); var files = Plugins.CompositeExporter.ExportToFile(reports, competitionName, Plugins.ResultExtenders); foreach (var file in files) logger.WriteLineInfo($" {file}"); logger.NewLine(); logger.WriteLineHeader("// * Detailed results *"); foreach (var report in reports) { logger.WriteLineInfo(report.Benchmark.Description); logger.WriteLineStatistic(report.GetTargetRuns().GetStats().ToTimeStr()); logger.NewLine(); } logger.WriteLineStatistic($"Total time: {globalStopwatch.Elapsed.TotalHours:00}:{globalStopwatch.Elapsed:mm\\:ss}"); logger.NewLine(); logger.WriteLineHeader("// * Summary *"); BenchmarkMarkdownExporter.Default.Export(reports, logger, Plugins.ResultExtenders); var warnings = Plugins.CompositeAnalyser.Analyze(reports).ToList(); if (warnings.Count > 0) { logger.NewLine(); logger.WriteLineError("// * Warnings * "); foreach (var warning in warnings) logger.WriteLineError($"{warning.Message}"); } logger.NewLine(); logger.WriteLineHeader("// ***** BenchmarkRunner: End *****"); return reports; }