public void BuildProjectFromPerfSuite(string name, ProjectPerfOperation operation) { string sourceProject = Path.Combine(@"C:\MSBPerf\3", name); var testDir = _testAssetsManager.CreateTestDirectory("Perf_" + name, identifier: operation.ToString()); FolderSnapshot.MirrorFiles(sourceProject, testDir.Path); // The generated projects target .NET Core 2.1, retarget them to .NET Core 2.0 foreach (var projFile in Directory.GetFiles(testDir.Path, "*.csproj", SearchOption.AllDirectories)) { var project = XDocument.Load(projFile); var ns = project.Root.Name.Namespace; // Find both TargetFramework and TargetFrameworks elements var targetFrameworkElements = project.Root.Elements(ns + "PropertyGroup").Elements("TargetFramework"); targetFrameworkElements = targetFrameworkElements.Concat(project.Root.Elements(ns + "PropertyGroup").Elements("TargetFrameworks")); foreach (var tfElement in targetFrameworkElements) { tfElement.Value = tfElement.Value.Replace("netcoreapp2.1", "netcoreapp2.0"); } project.Save(projFile); } TestProject(testDir.Path, name, operation); }
public void BuildRoslynCompilers(ProjectPerfOperation operation) { string sourceProject = @"C:\git\roslyn"; var testDir = _testAssetsManager.CreateTestDirectory("Perf_Roslyn", identifier: operation.ToString()); Console.WriteLine($"Mirroring {sourceProject} to {testDir.Path}..."); FolderSnapshot.MirrorFiles(sourceProject, testDir.Path); TestContext.Current.WriteGlobalJson(testDir.Path); Console.WriteLine("Done"); // Override global.json from repo File.Delete(Path.Combine(testDir.Path, "global.json")); // Run Roslyn's restore script var restoreCmd = new SdkCommandSpec() { FileName = Path.Combine(testDir.Path, "Restore.cmd"), WorkingDirectory = testDir.Path }; TestContext.Current.AddTestEnvironmentVariables(restoreCmd); restoreCmd.ToCommand().Execute().Should().Pass(); TestProject(Path.Combine(testDir.Path, "Compilers.sln"), "Roslyn", operation); }
private void TestProjectFromPerfTestSourceRepository( ProjectPerfOperation operation, string testName, string solutionDirectoryNameInPerfTestRepo, string projectDirectoryName = null) { string sourceProject = Path.Combine( TestContext.GetRepoRoot(), $".perftestsource/PerformanceTestProjects/{solutionDirectoryNameInPerfTestRepo}"); var testDir = _testAssetsManager.CreateTestDirectory( solutionDirectoryNameInPerfTestRepo, identifier: operation.ToString()); Console.WriteLine($"Mirroring {sourceProject} to {testDir}..."); FolderSnapshot.MirrorFiles(sourceProject, testDir.Path); TestContext.Current.WriteGlobalJson(testDir.Path); Console.WriteLine("Done"); string projectFolderOrFile = projectDirectoryName != null ? Path.Combine(testDir.Path, projectDirectoryName) : testDir.Path; TestProject(projectFolderOrFile, testName, operation); }
public static FolderSnapshot Create(string path) { FolderSnapshot folderSnapshot = new FolderSnapshot(); folderSnapshot.OriginalPath = path; folderSnapshot.BackupPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); MirrorFiles(folderSnapshot.OriginalPath, folderSnapshot.BackupPath); return(folderSnapshot); }
public void BuildWebLarge30(ProjectPerfOperation operation) { string sourceProject = Path.Combine(TestContext.GetRepoRoot(), ".perftestsource/PerformanceTestProjects/WebLarge30"); var testDir = _testAssetsManager.CreateTestDirectory("WebLarge30", identifier: operation.ToString()); Console.WriteLine($"Mirroring {sourceProject} to {testDir}..."); FolderSnapshot.MirrorFiles(sourceProject, testDir.Path); Console.WriteLine("Done"); TestProject(Path.Combine(testDir.Path, "mvc"), "Build Web Large 3.0", operation); }
public void Run([CallerMemberName] string callerName = null) { // Handle case where we're running inside VS (or via dotnet test), so the Main method hasn't run to initialize the perf harness // In the future we may want to do this via an xUnit fixture, which would also let us call the dispose method to write the results // afterwards if (_performanceHarness == null) { Program.HandlePerfArgs(new List <string>() { "--iterations", "1" }); } TestName = TestName ?? callerName; int currentIteration = 0; var durationTestModel = new ScenarioTestModel(TestName); durationTestModel.Performance.Metrics.Add(new MetricModel { Name = "ExecutionTime", DisplayName = "Execution Time", Unit = "ms" }); string testIdentifier = _performanceHarness.Configuration.RunId + "-" + ScenarioName + " - " + TestName; string testResultsFolder = Path.Combine(_performanceHarness.OutputDirectory, testIdentifier + "-traces"); if (!Directory.Exists(testResultsFolder)) { Directory.CreateDirectory(testResultsFolder); } using (FolderSnapshot snapshot = FolderSnapshot.Create(TestFolder)) { void PreIteration(ScenarioTest scenarioTest) { if (currentIteration > 0) { snapshot.Restore(); } // TODO: Optionally kill processes such as MSBuild.exe and VBCSCompiler.exe // We should always do this before the first iteration, but it should be configurable whether we // do it between iterations. This is because when testing "warm" / incremental builds, we would // expect the persistent processes to already be running and have already built the project } void PostIteration(ScenarioExecutionResult scenarioExecutionResult) { var elapsed = scenarioExecutionResult.ProcessExitInfo.ExitTime - scenarioExecutionResult.ProcessExitInfo.StartTime; var durationIteration = new IterationModel { Iteration = new Dictionary <string, double>() }; durationIteration.Iteration.Add(durationTestModel.Performance.Metrics[0].Name, elapsed.TotalMilliseconds); durationTestModel.Performance.IterationModels.Add(durationIteration); if (GetPerformanceSummary) { string performanceSummaryFileDestination = Path.Combine(testResultsFolder, $"{testIdentifier}({currentIteration}).txt"); File.Move(Path.Combine(TestFolder, "PerformanceSummary.txt"), performanceSummaryFileDestination); } if (GetBinLog) { string binlogDestination = Path.Combine(testResultsFolder, $"{testIdentifier}({currentIteration}).binlog"); File.Move(Path.Combine(TestFolder, "msbuild.binlog"), binlogDestination); } currentIteration++; } void PostRun(ScenarioBenchmark scenario) { } if (GetPerformanceSummary) { ProcessToMeasure.Arguments += " /flp9:PerformanceSummary;v=q;logfile=\"" + Path.Combine(TestFolder, "PerformanceSummary.txt") + "\""; } if (GetBinLog) { ProcessToMeasure.Arguments += " /bl:\"" + Path.Combine(TestFolder, "msbuild.binlog") + "\""; } var scenarioTestConfiguration = new ScenarioTestConfiguration(TimeSpan.FromMilliseconds(Timeout.TotalMilliseconds), ProcessToMeasure); scenarioTestConfiguration.Iterations = NumberOfIterations; scenarioTestConfiguration.PreIterationDelegate = PreIteration; scenarioTestConfiguration.PostIterationDelegate = PostIteration; scenarioTestConfiguration.SaveResults = false; scenarioTestConfiguration.Scenario = GetScenarioBenchmark(ScenarioName ?? TestName); scenarioTestConfiguration.Scenario.Tests.Add(durationTestModel); scenarioTestConfiguration.TestName = TestName; _performanceHarness.RunScenario(scenarioTestConfiguration, PostRun); } }