BenchmarkRunResult MeasureIterations(TestRun run, BenchmarkConfiguration config, ITestOutputHelper output) { // The XunitPerformanceHarness is hardcoded to log to the console. It would be nice if the output was configurable somehow // but in lieue of that we can redirect all console output with light hackery. using (var redirector = new ConsoleRedirector(output)) { // XunitPerformanceHarness expects to do the raw commandline parsing itself, but I really don't like that its default collection // metric requires the use of ETW. Getting an admin console or admin VS instance isn't where most people start, its // a small nuissance, and for these tests its often not needed/adds non-trivial overhead. I set the default to stopwatch if the // perf:collect argument hasn't been specified, but that sadly requires that I pre-parse, interpret, and then re-format all the // args to make that change :( // // In TestRun.ValidateMetricNames() I pre-check if ETW is going to be needed and give an error there rather than doing all the // test setup (~1 minute?) and then giving the error after the user has probably wandered away. That also relies on some of this // replicated command line parsing. string[] args = new string[] { "--perf:collect", string.Join("+", run.MetricNames), "--perf:outputdir", run.OutputDir, "--perf:runid", run.BenchviewRunId }; using (var harness = new XunitPerformanceHarness(args)) { ProcessStartInfo startInfo = new ProcessStartInfo(run.DotNetInstallation.DotNetExe, (ExePath + " " + CommandLineArguments).Trim()); startInfo.WorkingDirectory = WorkingDirPath; startInfo.RedirectStandardError = true; startInfo.RedirectStandardOutput = true; IEnumerable <KeyValuePair <string, string> > extraEnvVars = config.EnvironmentVariables.Concat(EnvironmentVariables).Append(new KeyValuePair <string, string>("DOTNET_MULTILEVEL_LOOKUP", "0")); foreach (KeyValuePair <string, string> kv in extraEnvVars) { startInfo.Environment[kv.Key] = kv.Value; } output.WriteLine("XUnitPerfHarness doesn't log env vars it uses to run processes. To workaround, logging them here:"); output.WriteLine($"Environment variables: {string.Join(", ", extraEnvVars.Select(kv => kv.Key + "=" + kv.Value))}"); output.WriteLine($"Working directory: \"{startInfo.WorkingDirectory}\""); output.WriteLine($"Command line: \"{startInfo.FileName}\" {startInfo.Arguments}"); BenchmarkRunResult result = new BenchmarkRunResult(this, config); StringBuilder stderr = new StringBuilder(); StringBuilder stdout = new StringBuilder(); var scenarioConfiguration = new ScenarioTestConfiguration(TimeSpan.FromMinutes(60), startInfo) { //XUnitPerformanceHarness writes files to disk starting with {runid}-{ScenarioBenchmarkName}-{TestName} TestName = (Name + "-" + config.Name).Replace(' ', '_'), Scenario = new ScenarioBenchmark("JitBench"), Iterations = run.Iterations, PreIterationDelegate = scenario => { stderr.Clear(); stdout.Clear(); scenario.Process.ErrorDataReceived += (object sender, DataReceivedEventArgs errorLine) => { if (!string.IsNullOrEmpty(errorLine.Data)) { stderr.AppendLine(errorLine.Data); redirector.WriteLine("STDERROR: " + errorLine.Data); } }; scenario.Process.OutputDataReceived += (object sender, DataReceivedEventArgs outputLine) => { stdout.AppendLine(outputLine.Data); redirector.WriteLine(outputLine.Data); }; }, PostIterationDelegate = scenarioResult => { result.IterationResults.Add(RecordIterationMetrics(scenarioResult, stdout.ToString(), stderr.ToString(), redirector)); } }; harness.RunScenario(scenarioConfiguration, sb => { BenchviewResultExporter.ConvertRunResult(sb, result); }); return(result); } } }
public static int Main(string[] args) { var options = BenchmarkOptions.Parse(args); bool benchmarkSpecified = false; foreach (Benchmark benchmark in Benchmarks) { if (options.BenchmarkNames.Contains(benchmark.Name, StringComparer.OrdinalIgnoreCase)) { benchmark.SetToRun(); benchmarkSpecified = true; break; } } var arguments = new List <string>(); string runId = ""; for (int i = 0; i < args.Length; i++) { if (string.Compare(args[i], "--perf:runid", true) == 0) { if (i + 1 < args.Length) { runId = args[++i]; } else { Console.WriteLine("Missing --perf:runid"); Console.WriteLine(BenchmarkOptions.Usage()); Environment.Exit(1); } } else { arguments.Add(args[i]); } } // Workspace is the ROOT of the coreclr tree. // If CORECLR_REPO is not set, the script assumes that the location of sandbox // is <path>\coreclr\sandbox. LinkBenchRoot = Directory.GetCurrentDirectory(); Workspace = Environment.GetEnvironmentVariable("CORECLR_REPO"); if (Workspace == null) { Workspace = Directory.GetParent(LinkBenchRoot).FullName; } if (Workspace == null) { Console.WriteLine("CORECLR_REPO not found"); return(-1); } string linkBenchSrcDir = Workspace + "\\tests\\src\\performance\\linkbench\\"; ScriptDir = linkBenchSrcDir + "scripts\\"; AssetsDir = linkBenchSrcDir + "assets\\"; Environment.SetEnvironmentVariable("LinkBenchRoot", LinkBenchRoot); Environment.SetEnvironmentVariable("__dotnet", LinkBenchRoot + "\\.Net\\dotnet.exe"); Environment.SetEnvironmentVariable("__dotnet2", LinkBenchRoot + "\\.Net2\\dotnet.exe"); PrintHeader("Update the build files to facilitate the link step."); if (options.DoSetup) { PrintHeader("Clone the benchmarks."); using (var setup = new Process()) { setup.StartInfo.FileName = ScriptDir + "clone.cmd"; setup.Start(); setup.WaitForExit(); if (setup.ExitCode != 0) { PrintHeader("Benchmark Setup failed"); return(-2); } } PrintHeader("Setup the benchmarks."); foreach (Benchmark benchmark in Benchmarks) { if (benchmark.doRun && benchmark.Setup != null) { benchmark.Setup(); } } } if (options.DoBuild) { // Run the setup Script, which clones, builds and links the benchmarks. using (var setup = new Process()) { setup.StartInfo.FileName = ScriptDir + "build.cmd"; setup.StartInfo.Arguments = AssetsDir; setup.Start(); setup.WaitForExit(); if (setup.ExitCode != 0) { Console.WriteLine("Benchmark build failed"); return(-3); } } } // Since this is a size measurement scenario, there are no iterations // to perform. So, create a process that does nothing, to satisfy XUnit. // All size measurements are performed PostRun() var emptyCmd = new ProcessStartInfo() { FileName = ScriptDir + "empty.cmd" }; for (int i = 0; i < Benchmarks.Length; i++) { CurrentBenchmark = Benchmarks[i]; if (!CurrentBenchmark.doRun) { continue; } var newArgs = new List <string>(arguments); newArgs.AddRange(new[] { "--perf:runid", $"{runId}-{CurrentBenchmark.Name}", }); Console.WriteLine($"{string.Join(" ", newArgs)}"); using (var h = new XunitPerformanceHarness(newArgs.ToArray())) { var configuration = new ScenarioTestConfiguration(new TimeSpan(2000000), emptyCmd) { Scenario = new ScenarioBenchmark(CurrentBenchmark.Name) { Namespace = "LinkBench" }, }; h.RunScenario(configuration, PostRun); } } return(0); }
static void Main(string[] args) { // The flag below is set to false to prevent the VBCSCompiler.exe hanging around // after the performance execution finished and preventing the deletion of the folder. Environment.SetEnvironmentVariable("UseSharedCompilation", "false"); var options = JitBenchHarnessOptions.Parse(args); SetupStatics(options); using (var h = new XunitPerformanceHarness(args)) { ProcessStartInfo startInfo = options.UseExistingSetup ? UseExistingSetup() : CreateNewSetup(); string scenarioName = "MusicStore"; if (!startInfo.Environment.ContainsKey("DOTNET_MULTILEVEL_LOOKUP")) { throw new InvalidOperationException("DOTNET_MULTILEVEL_LOOKUP was not defined."); } if (startInfo.Environment["DOTNET_MULTILEVEL_LOOKUP"] != "0") { throw new InvalidOperationException("DOTNET_MULTILEVEL_LOOKUP was not set to 0."); } if (options.EnableTiering) { startInfo.Environment.Add("COMPlus_EXPERIMENTAL_TieredCompilation", "1"); scenarioName += " Tiering"; } if (options.Minopts) { startInfo.Environment.Add("COMPlus_JITMinOpts", "1"); scenarioName += " Minopts"; } if (options.DisableR2R) { startInfo.Environment.Add("COMPlus_ReadyToRun", "0"); scenarioName += " NoR2R"; } if (options.DisableNgen) { startInfo.Environment.Add("COMPlus_ZapDisable", "1"); scenarioName += " NoNgen"; } PrintHeader($"Running scenario '{scenarioName}'"); var program = new JitBenchHarness(); try { var scenarioConfiguration = new ScenarioTestConfiguration(TimeSpan.FromMilliseconds(60000), startInfo) { Iterations = (int)options.Iterations, PreIterationDelegate = program.PreIteration, PostIterationDelegate = program.PostIteration, Scenario = new ScenarioBenchmark("JitBench"), }; var processesOfInterest = new string[] { "dotnet.exe", }; var modulesOfInterest = new string[] { "Anonymously Hosted DynamicMethods Assembly", "clrjit.dll", "coreclr.dll", "dotnet.exe", "MusicStore.dll", "ntoskrnl.exe", "System.Private.CoreLib.dll", "Unknown", }; if (!File.Exists(startInfo.FileName)) { throw new FileNotFoundException(startInfo.FileName); } if (!Directory.Exists(startInfo.WorkingDirectory)) { throw new DirectoryNotFoundException(startInfo.WorkingDirectory); } h.RunScenario(scenarioConfiguration, teardownDelegate: (ScenarioBenchmark scenarioBenchmark) => { program.PostRun(scenarioBenchmark, "MusicStore", processesOfInterest, modulesOfInterest); }); } catch { Console.WriteLine(program.StandardOutput); Console.WriteLine(program.StandardError); throw; } } }
private static void TestDir(XunitPerformanceHarness harness) { string commandName = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "dir" : "ls"; var testModel = new ScenarioTestModel(commandName); testModel.Performance.Metrics.Add(new MetricModel { Name = "ExecutionTime", DisplayName = "Execution Time", Unit = "ms" }); void PreIteration(ScenarioTest scenarioTest) { } void PostIteration(ScenarioExecutionResult scenarioExecutionResult) { var elapsed = scenarioExecutionResult.ProcessExitInfo.ExitTime - scenarioExecutionResult.ProcessExitInfo.StartTime; var iteration = new IterationModel { Iteration = new Dictionary <string, double>() }; iteration.Iteration.Add(testModel.Performance.Metrics[0].Name, elapsed.TotalMilliseconds); testModel.Performance.IterationModels.Add(iteration); } void PostRun(ScenarioBenchmark scenario) { } ProcessStartInfo processToMeasure; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { processToMeasure = new ProcessStartInfo("cmd.exe", $"/c {commandName}"); } else { processToMeasure = new ProcessStartInfo(commandName); } processToMeasure.RedirectStandardError = true; processToMeasure.RedirectStandardOutput = true; var scenarioTestConfiguration = new ScenarioTestConfiguration(Timeout, processToMeasure) { Iterations = Iterations, PreIterationDelegate = PreIteration, PostIterationDelegate = PostIteration, Scenario = new ScenarioBenchmark("ExecuteCommand") }; scenarioTestConfiguration.Scenario.Tests.Add(testModel); scenarioTestConfiguration.TestName = commandName; harness.RunScenario(scenarioTestConfiguration, PostRun); }
public void Run([CallerMemberName] string callerName = null) { // Handle case where we're running inside VS (or via dotnet test), so the Main method hasn't run to initialize the perf harness // In the future we may want to do this via an xUnit fixture, which would also let us call the dispose method to write the results // afterwards if (_performanceHarness == null) { Program.HandlePerfArgs(new List <string>() { "--iterations", "1" }); } TestName = TestName ?? callerName; int currentIteration = 0; var durationTestModel = new ScenarioTestModel(TestName); durationTestModel.Performance.Metrics.Add(new MetricModel { Name = "ExecutionTime", DisplayName = "Execution Time", Unit = "ms" }); string testIdentifier = _performanceHarness.Configuration.RunId + "-" + ScenarioName + " - " + TestName; string testResultsFolder = Path.Combine(_performanceHarness.OutputDirectory, testIdentifier + "-traces"); if (!Directory.Exists(testResultsFolder)) { Directory.CreateDirectory(testResultsFolder); } using (FolderSnapshot snapshot = FolderSnapshot.Create(TestFolder)) { void PreIteration(ScenarioTest scenarioTest) { if (currentIteration > 0) { snapshot.Restore(); } // TODO: Optionally kill processes such as MSBuild.exe and VBCSCompiler.exe // We should always do this before the first iteration, but it should be configurable whether we // do it between iterations. This is because when testing "warm" / incremental builds, we would // expect the persistent processes to already be running and have already built the project } void PostIteration(ScenarioExecutionResult scenarioExecutionResult) { var elapsed = scenarioExecutionResult.ProcessExitInfo.ExitTime - scenarioExecutionResult.ProcessExitInfo.StartTime; var durationIteration = new IterationModel { Iteration = new Dictionary <string, double>() }; durationIteration.Iteration.Add(durationTestModel.Performance.Metrics[0].Name, elapsed.TotalMilliseconds); durationTestModel.Performance.IterationModels.Add(durationIteration); if (GetPerformanceSummary) { string performanceSummaryFileDestination = Path.Combine(testResultsFolder, $"{testIdentifier}({currentIteration}).txt"); File.Move(Path.Combine(TestFolder, "PerformanceSummary.txt"), performanceSummaryFileDestination); } if (GetBinLog) { string binlogDestination = Path.Combine(testResultsFolder, $"{testIdentifier}({currentIteration}).binlog"); File.Move(Path.Combine(TestFolder, "msbuild.binlog"), binlogDestination); } currentIteration++; } void PostRun(ScenarioBenchmark scenario) { } if (GetPerformanceSummary) { ProcessToMeasure.Arguments += " /flp9:PerformanceSummary;v=q;logfile=\"" + Path.Combine(TestFolder, "PerformanceSummary.txt") + "\""; } if (GetBinLog) { ProcessToMeasure.Arguments += " /bl:\"" + Path.Combine(TestFolder, "msbuild.binlog") + "\""; } var scenarioTestConfiguration = new ScenarioTestConfiguration(TimeSpan.FromMilliseconds(Timeout.TotalMilliseconds), ProcessToMeasure); scenarioTestConfiguration.Iterations = NumberOfIterations; scenarioTestConfiguration.PreIterationDelegate = PreIteration; scenarioTestConfiguration.PostIterationDelegate = PostIteration; scenarioTestConfiguration.SaveResults = false; scenarioTestConfiguration.Scenario = GetScenarioBenchmark(ScenarioName ?? TestName); scenarioTestConfiguration.Scenario.Tests.Add(durationTestModel); scenarioTestConfiguration.TestName = TestName; _performanceHarness.RunScenario(scenarioTestConfiguration, PostRun); } }