List <ResultTableRowModel> BuildRowModels(Func <Benchmark, Metric, bool> primaryMetricSelector) { List <ResultTableRowModel> rows = new List <ResultTableRowModel>(); foreach (Benchmark benchmark in Benchmarks) { BenchmarkRunResult canonResult = BenchmarkRunResults.Where(r => r.Benchmark == benchmark).FirstOrDefault(); if (canonResult == null || canonResult.IterationResults == null || canonResult.IterationResults.Count == 0) { continue; } IterationResult canonIteration = canonResult.IterationResults[0]; foreach (Metric metric in canonIteration.Measurements.Keys) { if (primaryMetricSelector(benchmark, metric)) { rows.Add(new ResultTableRowModel() { Benchmark = benchmark, Metric = metric }); } } } return(rows); }
static List <Metric> CollectMetrics(BenchmarkRunResult runResult) { List <Metric> metrics = new List <Metric>(); foreach (IterationResult iterationResult in runResult.IterationResults) { foreach (KeyValuePair <Metric, double> measurement in iterationResult.Measurements) { if (!metrics.Contains(measurement.Key)) { metrics.Add(measurement.Key); } } } return(metrics); }
static ScenarioTestModel[] ConvertRunResult(BenchmarkRunResult runResult) { List <ScenarioTestModel> testModels = new List <ScenarioTestModel>(); string name = runResult.Benchmark.Name; List <Metric> metrics = CollectMetrics(runResult); foreach (Metric m in metrics.ToArray()) { if (runResult.Benchmark.TryGetBenchviewCustomMetricReporting(m, out Metric newMetric, out string newScenarioModelName)) { metrics.Remove(m); testModels.Add(ConvertRunResult(runResult, new Metric[] { newMetric }, oldMetric => m.Equals(oldMetric) ? newMetric : default(Metric), name, newScenarioModelName)); } } testModels.Insert(0, ConvertRunResult(runResult, metrics, oldMetric => metrics.Contains(oldMetric) ? oldMetric : default(Metric), null, name)); return(testModels.ToArray()); }
static ScenarioTestModel ConvertRunResult(BenchmarkRunResult runResult, IEnumerable <Metric> metrics, Func <Metric, Metric> metricMapping, string scenarioModelNamespace, string scenarioModelName) { var testModel = new ScenarioTestModel(scenarioModelName); testModel.Namespace = scenarioModelNamespace; testModel.Performance = new PerformanceModel(); testModel.Performance.Metrics = new List <MetricModel>(); testModel.Performance.IterationModels = new List <IterationModel>(); foreach (var iterationResult in runResult.IterationResults) { testModel.Performance.IterationModels.Add(ConvertIterationResult(iterationResult, metricMapping)); } foreach (var metric in metrics) { testModel.Performance.Metrics.Add(new MetricModel() { DisplayName = metric.Name, Name = metric.Name, Unit = metric.Unit }); } return(testModel); }
BenchmarkRunResult MeasureIterations(TestRun run, BenchmarkConfiguration config, ITestOutputHelper output) { // The XunitPerformanceHarness is hardcoded to log to the console. It would be nice if the output was configurable somehow // but in lieue of that we can redirect all console output with light hackery. using (var redirector = new ConsoleRedirector(output)) { // XunitPerformanceHarness expects to do the raw commandline parsing itself, but I really don't like that its default collection // metric requires the use of ETW. Getting an admin console or admin VS instance isn't where most people start, its // a small nuissance, and for these tests its often not needed/adds non-trivial overhead. I set the default to stopwatch if the // perf:collect argument hasn't been specified, but that sadly requires that I pre-parse, interpret, and then re-format all the // args to make that change :( // // In TestRun.ValidateMetricNames() I pre-check if ETW is going to be needed and give an error there rather than doing all the // test setup (~1 minute?) and then giving the error after the user has probably wandered away. That also relies on some of this // replicated command line parsing. string[] args = new string[] { "--perf:collect", string.Join("+", run.MetricNames), "--perf:outputdir", run.OutputDir, "--perf:runid", run.BenchviewRunId }; using (var harness = new XunitPerformanceHarness(args)) { ProcessStartInfo startInfo = new ProcessStartInfo(run.DotNetInstallation.DotNetExe, (ExePath + " " + CommandLineArguments).Trim()); startInfo.WorkingDirectory = WorkingDirPath; startInfo.RedirectStandardError = true; startInfo.RedirectStandardOutput = true; IEnumerable <KeyValuePair <string, string> > extraEnvVars = config.EnvironmentVariables.Concat(EnvironmentVariables).Append(new KeyValuePair <string, string>("DOTNET_MULTILEVEL_LOOKUP", "0")); foreach (KeyValuePair <string, string> kv in extraEnvVars) { startInfo.Environment[kv.Key] = kv.Value; } output.WriteLine("XUnitPerfHarness doesn't log env vars it uses to run processes. To workaround, logging them here:"); output.WriteLine($"Environment variables: {string.Join(", ", extraEnvVars.Select(kv => kv.Key + "=" + kv.Value))}"); output.WriteLine($"Working directory: \"{startInfo.WorkingDirectory}\""); output.WriteLine($"Command line: \"{startInfo.FileName}\" {startInfo.Arguments}"); BenchmarkRunResult result = new BenchmarkRunResult(this, config); StringBuilder stderr = new StringBuilder(); StringBuilder stdout = new StringBuilder(); var scenarioConfiguration = new ScenarioTestConfiguration(TimeSpan.FromMinutes(60), startInfo) { //XUnitPerformanceHarness writes files to disk starting with {runid}-{ScenarioBenchmarkName}-{TestName} TestName = (Name + "-" + config.Name).Replace(' ', '_'), Scenario = new ScenarioBenchmark("JitBench"), Iterations = run.Iterations, PreIterationDelegate = scenario => { stderr.Clear(); stdout.Clear(); scenario.Process.ErrorDataReceived += (object sender, DataReceivedEventArgs errorLine) => { if (!string.IsNullOrEmpty(errorLine.Data)) { stderr.AppendLine(errorLine.Data); redirector.WriteLine("STDERROR: " + errorLine.Data); } }; scenario.Process.OutputDataReceived += (object sender, DataReceivedEventArgs outputLine) => { stdout.AppendLine(outputLine.Data); redirector.WriteLine(outputLine.Data); }; }, PostIterationDelegate = scenarioResult => { result.IterationResults.Add(RecordIterationMetrics(scenarioResult, stdout.ToString(), stderr.ToString(), redirector)); } }; harness.RunScenario(scenarioConfiguration, sb => { BenchviewResultExporter.ConvertRunResult(sb, result); }); return(result); } } }
public static void ConvertRunResult(ScenarioBenchmark scenario, BenchmarkRunResult runResult) { scenario.Tests = new List <ScenarioTestModel>(); scenario.Tests.AddRange(ConvertRunResult(runResult)); }