public void BuildProjectFromPerfSuite(string name, ProjectPerfOperation operation)
{
string sourceProject = Path.Combine(@"C:\MSBPerf\3", name);
var testDir = _testAssetsManager.CreateTestDirectory("Perf_" + name, identifier: operation.ToString());
FolderSnapshot.MirrorFiles(sourceProject, testDir.Path);
// The generated projects target .NET Core 2.1, retarget them to .NET Core 2.0
foreach (var projFile in Directory.GetFiles(testDir.Path, "*.csproj", SearchOption.AllDirectories))
{
var project = XDocument.Load(projFile);
var ns = project.Root.Name.Namespace;
// Find both TargetFramework and TargetFrameworks elements
var targetFrameworkElements = project.Root.Elements(ns + "PropertyGroup").Elements("TargetFramework");
targetFrameworkElements = targetFrameworkElements.Concat(project.Root.Elements(ns + "PropertyGroup").Elements("TargetFrameworks"));
foreach (var tfElement in targetFrameworkElements)
{
tfElement.Value = tfElement.Value.Replace("netcoreapp2.1", "netcoreapp2.0");
}
project.Save(projFile);
}
TestProject(testDir.Path, name, operation);
}
public void BuildRoslynCompilers(ProjectPerfOperation operation)
{
string sourceProject = @"C:\git\roslyn";
var testDir = _testAssetsManager.CreateTestDirectory("Perf_Roslyn", identifier: operation.ToString());
Console.WriteLine($"Mirroring {sourceProject} to {testDir.Path}...");
FolderSnapshot.MirrorFiles(sourceProject, testDir.Path);
TestContext.Current.WriteGlobalJson(testDir.Path);
Console.WriteLine("Done");
// Override global.json from repo
File.Delete(Path.Combine(testDir.Path, "global.json"));
// Run Roslyn's restore script
var restoreCmd = new SdkCommandSpec()
{
FileName = Path.Combine(testDir.Path, "Restore.cmd"),
WorkingDirectory = testDir.Path
};
TestContext.Current.AddTestEnvironmentVariables(restoreCmd);
restoreCmd.ToCommand().Execute().Should().Pass();
TestProject(Path.Combine(testDir.Path, "Compilers.sln"), "Roslyn", operation);
}
private void TestProjectFromPerfTestSourceRepository(
ProjectPerfOperation operation,
string testName,
string solutionDirectoryNameInPerfTestRepo,
string projectDirectoryName = null)
{
string sourceProject = Path.Combine(
TestContext.GetRepoRoot(),
$".perftestsource/PerformanceTestProjects/{solutionDirectoryNameInPerfTestRepo}");
var testDir = _testAssetsManager.CreateTestDirectory(
solutionDirectoryNameInPerfTestRepo,
identifier: operation.ToString());
Console.WriteLine($"Mirroring {sourceProject} to {testDir}...");
FolderSnapshot.MirrorFiles(sourceProject, testDir.Path);
TestContext.Current.WriteGlobalJson(testDir.Path);
Console.WriteLine("Done");
string projectFolderOrFile =
projectDirectoryName != null
? Path.Combine(testDir.Path, projectDirectoryName)
: testDir.Path;
TestProject(projectFolderOrFile, testName, operation);
}
public static FolderSnapshot Create(string path)
{
FolderSnapshot folderSnapshot = new FolderSnapshot();
folderSnapshot.OriginalPath = path;
folderSnapshot.BackupPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
MirrorFiles(folderSnapshot.OriginalPath, folderSnapshot.BackupPath);
return(folderSnapshot);
}
public void BuildWebLarge30(ProjectPerfOperation operation)
{
string sourceProject = Path.Combine(TestContext.GetRepoRoot(), ".perftestsource/PerformanceTestProjects/WebLarge30");
var testDir = _testAssetsManager.CreateTestDirectory("WebLarge30", identifier: operation.ToString());
Console.WriteLine($"Mirroring {sourceProject} to {testDir}...");
FolderSnapshot.MirrorFiles(sourceProject, testDir.Path);
Console.WriteLine("Done");
TestProject(Path.Combine(testDir.Path, "mvc"), "Build Web Large 3.0", operation);
}
public void Run([CallerMemberName] string callerName = null)
{
// Handle case where we're running inside VS (or via dotnet test), so the Main method hasn't run to initialize the perf harness
// In the future we may want to do this via an xUnit fixture, which would also let us call the dispose method to write the results
// afterwards
if (_performanceHarness == null)
{
Program.HandlePerfArgs(new List <string>()
{
"--iterations",
"1"
});
}
TestName = TestName ?? callerName;
int currentIteration = 0;
var durationTestModel = new ScenarioTestModel(TestName);
durationTestModel.Performance.Metrics.Add(new MetricModel
{
Name = "ExecutionTime",
DisplayName = "Execution Time",
Unit = "ms"
});
string testIdentifier = _performanceHarness.Configuration.RunId + "-" + ScenarioName + " - " + TestName;
string testResultsFolder = Path.Combine(_performanceHarness.OutputDirectory, testIdentifier + "-traces");
if (!Directory.Exists(testResultsFolder))
{
Directory.CreateDirectory(testResultsFolder);
}
using (FolderSnapshot snapshot = FolderSnapshot.Create(TestFolder))
{
void PreIteration(ScenarioTest scenarioTest)
{
if (currentIteration > 0)
{
snapshot.Restore();
}
// TODO: Optionally kill processes such as MSBuild.exe and VBCSCompiler.exe
// We should always do this before the first iteration, but it should be configurable whether we
// do it between iterations. This is because when testing "warm" / incremental builds, we would
// expect the persistent processes to already be running and have already built the project
}
void PostIteration(ScenarioExecutionResult scenarioExecutionResult)
{
var elapsed = scenarioExecutionResult.ProcessExitInfo.ExitTime - scenarioExecutionResult.ProcessExitInfo.StartTime;
var durationIteration = new IterationModel
{
Iteration = new Dictionary <string, double>()
};
durationIteration.Iteration.Add(durationTestModel.Performance.Metrics[0].Name, elapsed.TotalMilliseconds);
durationTestModel.Performance.IterationModels.Add(durationIteration);
if (GetPerformanceSummary)
{
string performanceSummaryFileDestination = Path.Combine(testResultsFolder, $"{testIdentifier}({currentIteration}).txt");
File.Move(Path.Combine(TestFolder, "PerformanceSummary.txt"), performanceSummaryFileDestination);
}
if (GetBinLog)
{
string binlogDestination = Path.Combine(testResultsFolder, $"{testIdentifier}({currentIteration}).binlog");
File.Move(Path.Combine(TestFolder, "msbuild.binlog"), binlogDestination);
}
currentIteration++;
}
void PostRun(ScenarioBenchmark scenario)
{
}
if (GetPerformanceSummary)
{
ProcessToMeasure.Arguments += " /flp9:PerformanceSummary;v=q;logfile=\"" + Path.Combine(TestFolder, "PerformanceSummary.txt") + "\"";
}
if (GetBinLog)
{
ProcessToMeasure.Arguments += " /bl:\"" + Path.Combine(TestFolder, "msbuild.binlog") + "\"";
}
var scenarioTestConfiguration = new ScenarioTestConfiguration(TimeSpan.FromMilliseconds(Timeout.TotalMilliseconds), ProcessToMeasure);
scenarioTestConfiguration.Iterations = NumberOfIterations;
scenarioTestConfiguration.PreIterationDelegate = PreIteration;
scenarioTestConfiguration.PostIterationDelegate = PostIteration;
scenarioTestConfiguration.SaveResults = false;
scenarioTestConfiguration.Scenario = GetScenarioBenchmark(ScenarioName ?? TestName);
scenarioTestConfiguration.Scenario.Tests.Add(durationTestModel);
scenarioTestConfiguration.TestName = TestName;
_performanceHarness.RunScenario(scenarioTestConfiguration, PostRun);
}
}
