public EtwPerformanceMetricLogger(XunitPerformanceProject project, Program program) { _etlPath = Path.Combine(project.OutputDir, project.OutputBaseFileName + ".etl"); _program = program; _project = project; var diagnosticMessageSink = new ConsoleReporter(); foreach (var assembly in project.Assemblies) { program.PrintIfVerbose($"Discovering tests for {assembly.AssemblyFilename}."); // Note: We do not use shadowCopy because that creates a new AppDomain which can cause // assembly load failures with delay-signed or "fake signed" assemblies. using (var controller = new XunitFrontController( assemblyFileName: assembly.AssemblyFilename, shadowCopy: false, appDomainSupport: AppDomainSupport.Denied, diagnosticMessageSink: new ConsoleDiagnosticsMessageVisitor()) ) using (var discoveryVisitor = new PerformanceTestDiscoveryVisitor(assembly, project.Filters, diagnosticMessageSink)) { controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: TestFrameworkOptions.ForDiscovery()); discoveryVisitor.Finished.WaitOne(); _tests.AddRange(discoveryVisitor.Tests); } } program.PrintIfVerbose($"Discovered a total of {_tests.Count} tests."); }
private static void AddBaseline(XunitPerformanceProject project, string assembly) { project.AddBaseline(new XunitProjectAssembly { AssemblyFilename = Path.GetFullPath(assembly), }); }
private static XunitPerformanceProject GetProjectFile(List <Tuple <string, string> > assemblies) { var result = new XunitPerformanceProject(); foreach (var assembly in assemblies) { result.Add(new XunitProjectAssembly { AssemblyFilename = Path.GetFullPath(assembly.Item1), ConfigFilename = assembly.Item2 != null ? Path.GetFullPath(assembly.Item2) : null, }); } return(result); }
private static XunitPerformanceProject GetProjectFile(List<Tuple<string, string>> assemblies) { var result = new XunitPerformanceProject(); foreach (var assembly in assemblies) result.Add(new XunitProjectAssembly { AssemblyFilename = Path.GetFullPath(assembly.Item1), ConfigFilename = assembly.Item2 != null ? Path.GetFullPath(assembly.Item2) : null, }); return result; }
protected abstract IPerformanceMetricLogger GetPerformanceMetricLogger(XunitPerformanceProject project);
private void RunTests(XunitPerformanceProject project) { string xmlPath = Path.Combine(project.OutputDir, project.OutputBaseFileName + ".xml"); var commandLineArgs = new StringBuilder(); if (!string.IsNullOrEmpty(project.RunnerHost)) { commandLineArgs.Append("\""); commandLineArgs.Append(project.RunnerCommand); commandLineArgs.Append("\" "); } foreach (var assembly in project.Assemblies) { commandLineArgs.Append("\""); commandLineArgs.Append(assembly.AssemblyFilename); commandLineArgs.Append("\" "); } foreach (var testClass in project.Filters.IncludedClasses) { commandLineArgs.Append("-class "); commandLineArgs.Append(testClass); commandLineArgs.Append(" "); } foreach (var testMethod in project.Filters.IncludedMethods) { commandLineArgs.Append("-method "); commandLineArgs.Append(testMethod); commandLineArgs.Append(" "); } foreach (var trait in project.Filters.IncludedTraits) { foreach (var traitValue in trait.Value) { commandLineArgs.Append("-trait \""); commandLineArgs.Append(trait.Key); commandLineArgs.Append("="); commandLineArgs.Append(traitValue); commandLineArgs.Append("\" "); } } foreach (var trait in project.Filters.ExcludedTraits) { foreach (var traitValue in trait.Value) { commandLineArgs.Append("-notrait \""); commandLineArgs.Append(trait.Key); commandLineArgs.Append("="); commandLineArgs.Append(traitValue); commandLineArgs.Append("\" "); } } if (!string.IsNullOrEmpty(project.RunnerArgs)) { commandLineArgs.Append(project.RunnerArgs); commandLineArgs.Append(" "); } commandLineArgs.Append("-xml \""); commandLineArgs.Append(xmlPath); commandLineArgs.Append("\""); ProcessStartInfo startInfo = new ProcessStartInfo() { FileName = project.RunnerHost ?? project.RunnerCommand, Arguments = commandLineArgs.ToString(), UseShellExecute = false, }; startInfo.Environment["XUNIT_PERFORMANCE_RUN_ID"] = project.RunId; startInfo.Environment["XUNIT_PERFORMANCE_MIN_ITERATION"] = "10"; startInfo.Environment["XUNIT_PERFORMANCE_MAX_ITERATION"] = "1000"; startInfo.Environment["XUNIT_PERFORMANCE_MAX_TOTAL_MILLISECONDS"] = "1000"; startInfo.Environment["COMPLUS_gcConcurrent"] = "0"; startInfo.Environment["COMPLUS_gcServer"] = "0"; var logger = GetPerformanceMetricLogger(project); using (logger.StartLogging(startInfo)) { PrintIfVerbose($@"Launching runner: Runner: {startInfo.FileName} Arguments: {startInfo.Arguments}"); try { using (var proc = Process.Start(startInfo)) { proc.EnableRaisingEvents = true; proc.WaitForExit(); } } catch (Exception ex) { throw new Exception($"Could not launch the test runner, {startInfo.FileName}", innerException: ex); } } using (var evaluationContext = logger.GetReader()) { var xmlDoc = XDocument.Load(xmlPath); foreach (var testElem in xmlDoc.Descendants("test")) { var testName = testElem.Attribute("name").Value; var perfElem = new XElement("performance", new XAttribute("runid", project.RunId), new XAttribute("etl", Path.GetFullPath(evaluationContext.LogPath))); testElem.Add(perfElem); var metrics = evaluationContext.GetMetrics(testName); if (metrics != null) { var metricsElem = new XElement("metrics"); perfElem.Add(metricsElem); foreach (var metric in metrics) metricsElem.Add(new XElement(metric.Id, new XAttribute("displayName", metric.DisplayName), new XAttribute("unit", metric.Unit))); } var iterations = evaluationContext.GetValues(testName); if (iterations != null) { var iterationsElem = new XElement("iterations"); perfElem.Add(iterationsElem); for (int i = 0; i < iterations.Count; i++) { var iteration = iterations[i]; if (iteration != null) { var iterationElem = new XElement("iteration", new XAttribute("index", i)); iterationsElem.Add(iterationElem); foreach (var value in iteration) iterationElem.Add(new XAttribute(value.Key, value.Value.ToString("R"))); } } } } using (var xmlFile = File.Create(xmlPath)) xmlDoc.Save(xmlFile); } }
public CSVMetricLogger(XunitPerformanceProject project) { _csvPath = Path.GetFullPath(Path.Combine(project.OutputDir, project.RunId + ".csv")); }
internal void RunTests(XunitPerformanceProject project) { string xmlPath = Path.Combine(project.OutputDir, project.OutputBaseFileName + ".xml"); string scenarioRangesPath = Path.Combine(project.OutputDir, "ScenarioRanges.txt"); string zipPath = Path.Combine(project.OutputDir, project.OutputBaseFileName + ".etl.zip"); var commandLineArgs = new StringBuilder(); if (!string.IsNullOrEmpty(project.RunnerHost)) { commandLineArgs.Append("\""); commandLineArgs.Append(project.RunnerCommand); commandLineArgs.Append("\" "); } foreach (var assembly in project.Assemblies) { commandLineArgs.Append("\""); commandLineArgs.Append(assembly.AssemblyFilename); commandLineArgs.Append("\" "); } foreach (var testClass in project.Filters.IncludedClasses) { commandLineArgs.Append("-class "); commandLineArgs.Append(testClass); commandLineArgs.Append(" "); } foreach (var testMethod in project.Filters.IncludedMethods) { commandLineArgs.Append("-method "); commandLineArgs.Append(testMethod); commandLineArgs.Append(" "); } foreach (var trait in project.Filters.IncludedTraits) { foreach (var traitValue in trait.Value) { commandLineArgs.Append("-trait \""); commandLineArgs.Append(trait.Key); commandLineArgs.Append("="); commandLineArgs.Append(traitValue); commandLineArgs.Append("\" "); } } foreach (var trait in project.Filters.ExcludedTraits) { foreach (var traitValue in trait.Value) { commandLineArgs.Append("-notrait \""); commandLineArgs.Append(trait.Key); commandLineArgs.Append("="); commandLineArgs.Append(traitValue); commandLineArgs.Append("\" "); } } if (!string.IsNullOrEmpty(project.RunnerArgs)) { commandLineArgs.Append(project.RunnerArgs); commandLineArgs.Append(" "); } commandLineArgs.Append("-xml \""); commandLineArgs.Append(xmlPath); commandLineArgs.Append("\""); ProcessStartInfo startInfo = new ProcessStartInfo() { FileName = project.RunnerHost ?? project.RunnerCommand, Arguments = commandLineArgs.ToString(), UseShellExecute = false, }; startInfo.Environment["XUNIT_PERFORMANCE_RUN_ID"] = project.RunId; startInfo.Environment["XUNIT_PERFORMANCE_MIN_ITERATION"] = RunConfiguration.XUNIT_PERFORMANCE_MAX_ITERATION.ToString(); startInfo.Environment["XUNIT_PERFORMANCE_MAX_ITERATION"] = RunConfiguration.XUNIT_PERFORMANCE_MIN_ITERATION.ToString(); startInfo.Environment["XUNIT_PERFORMANCE_MAX_TOTAL_MILLISECONDS"] = RunConfiguration.XUNIT_PERFORMANCE_MAX_TOTAL_MILLISECONDS.ToString(); startInfo.Environment["COMPLUS_gcConcurrent"] = "0"; startInfo.Environment["COMPLUS_gcServer"] = "0"; if (project.UseLocalUser) { startInfo.Domain = project.runComputer; startInfo.UserName = project.runCredentialsUsername; startInfo.Password = project.runCredentialsPassword; startInfo.LoadUserProfile = true; foreach (var envvar in project.runEnvVars) { startInfo.Environment[envvar.Key.ToString()] = envvar.Value.ToString(); } } var logger = GetPerformanceMetricLogger(project); using (logger.StartLogging(startInfo)) { PrintIfVerbose($@"Launching runner: Runner: {startInfo.FileName} Arguments: {startInfo.Arguments}"); try { using (var proc = Process.Start(startInfo)) { proc.EnableRaisingEvents = true; proc.WaitForExit(); } } catch (Exception ex) { throw new Exception($"Could not launch the test runner, {startInfo.FileName}", innerException: ex); } } List<ScenarioRange> ScenarioRanges; using (var evaluationContext = logger.GetReader()) { ScenarioRanges = evaluationContext.GetScenarioRanges(); var xmlDoc = XDocument.Load(xmlPath); foreach (var assembly in xmlDoc.Descendants("assembly")) // create MetricDegradeBars section { var MetricDegradeBars = new XElement("MetricDegradeBars"); assembly.AddFirst(MetricDegradeBars); foreach (var testElem in assembly.Descendants("test")) { var testName = testElem.Attribute("name").Value; var perfElem = new XElement("performance", new XAttribute("runid", project.RunId), new XAttribute("etl", Path.GetFullPath(evaluationContext.LogPath))); testElem.Add(perfElem); var metrics = evaluationContext.GetMetrics(testName); if (metrics != null) { var metricsElem = new XElement("metrics"); perfElem.Add(metricsElem); foreach (var metric in metrics) { switch (metric.Unit) { case PerformanceMetricUnits.ListCount: metricsElem.Add(new XElement(metric.Id, new XAttribute("displayName", metric.DisplayName), new XAttribute("unit", PerformanceMetricUnits.List))); metricsElem.Add(new XElement(metric.Id + "Count", new XAttribute("displayName", metric.DisplayName + " Count"), new XAttribute("unit", PerformanceMetricUnits.Count))); CreateMetricDegradeBars(MetricDegradeBars, metric.Id + "Count"); break; case PerformanceMetricUnits.ListBytes: metricsElem.Add(new XElement(metric.Id, new XAttribute("displayName", metric.DisplayName), new XAttribute("unit", PerformanceMetricUnits.List))); metricsElem.Add(new XElement(metric.Id + "Bytes", new XAttribute("displayName", metric.DisplayName + " Bytes"), new XAttribute("unit", PerformanceMetricUnits.Bytes))); CreateMetricDegradeBars(MetricDegradeBars, metric.Id + "Bytes"); break; case PerformanceMetricUnits.ListCountBytes: metricsElem.Add(new XElement(metric.Id, new XAttribute("displayName", metric.DisplayName), new XAttribute("unit", PerformanceMetricUnits.List))); metricsElem.Add(new XElement(metric.Id + "Bytes", new XAttribute("displayName", metric.DisplayName + " Bytes"), new XAttribute("unit", PerformanceMetricUnits.Bytes))); metricsElem.Add(new XElement(metric.Id + "Count", new XAttribute("displayName", metric.DisplayName + " Count"), new XAttribute("unit", PerformanceMetricUnits.Count))); CreateMetricDegradeBars(MetricDegradeBars, metric.Id + "Count"); CreateMetricDegradeBars(MetricDegradeBars, metric.Id + "Bytes"); break; default: metricsElem.Add(new XElement(metric.Id, new XAttribute("displayName", metric.DisplayName), new XAttribute("unit", metric.Unit))); if (metric.Unit != PerformanceMetricUnits.List) CreateMetricDegradeBars(MetricDegradeBars, metric.Id); break; } } } var iterations = evaluationContext.GetValues(testName); if (iterations != null) { var iterationsElem = new XElement("iterations"); perfElem.Add(iterationsElem); for (int i = 0; i < iterations.Count; i++) { var iteration = iterations[i]; if (iteration != null) { var iterationElem = new XElement("iteration", new XAttribute("index", i)); iterationsElem.Add(iterationElem); foreach (var value in iteration) { double result; if (double.TryParse(value.Value.ToString(), out result)) { // result is a double, add it as an attribute iterationElem.Add(new XAttribute(value.Key, result.ToString("R"))); } else // result is a list, add the list as a new element { ListMetricInfo listMetricInfo = (ListMetricInfo)value.Value; if (listMetricInfo.hasCount) { string metricName = value.Key + "Count"; iterationElem.Add(new XAttribute(metricName, listMetricInfo.count.ToString())); } if (listMetricInfo.hasBytes) { string metricName = value.Key + "Bytes"; iterationElem.Add(new XAttribute(metricName, listMetricInfo.bytes.ToString())); } var listResult = new XElement("ListResult"); listResult.Add(new XAttribute("Name", value.Key)); listResult.Add(new XAttribute("Iteration", i)); iterationElem.Add(listResult); foreach (ListMetricInfo.Metrics listMetric in listMetricInfo.MetricList) { var ListMetric = new XElement("ListMetric"); ListMetric.Add(new XAttribute("Name", listMetric.Name)); ListMetric.Add(new XAttribute("Unit", listMetric.Unit)); ListMetric.Add(new XAttribute("Type", listMetric.Type.Name)); listResult.Add(ListMetric); } foreach (var listItem in listMetricInfo.Items.OrderByDescending(key => key.Value.Size)) { var ListItem = new XElement("ListItem"); ListItem.Add(new XAttribute("Name", listItem.Key)); ListItem.Add(new XAttribute("Size", listItem.Value.Size)); ListItem.Add(new XAttribute("Count", listItem.Value.Count)); listResult.Add(ListItem); } } } } } } } } // Create xunit results: runID.xml using (var xmlFile = File.Create(xmlPath)) { System.Xml.XmlWriterSettings settings = new System.Xml.XmlWriterSettings(); settings.CheckCharacters = false; settings.Indent = true; settings.IndentChars = " "; using (System.Xml.XmlWriter writer = System.Xml.XmlWriter.Create(xmlFile, settings)) xmlDoc.Save(writer); } // Create ScenarioRanges.txt using (var scenarioRangesFile = new StreamWriter(File.Create(scenarioRangesPath))) { scenarioRangesFile.WriteLine("ScenarioName,Scenario Start (in ms),Scenario Stop (in ms),PerfView start/stop range (for copy/paste)"); foreach(var scenarioRange in ScenarioRanges) { scenarioRangesFile.WriteLine($"{scenarioRange.ScenarioName},{scenarioRange.ScenarioStartTime},{scenarioRange.ScenarioStopTime},{scenarioRange.ScenarioStartTime} {scenarioRange.ScenarioStopTime}"); } } // Create PerformanceTempResults - runID.xml string tempResultsPath = Consumption.FormatXML.formatXML(xmlPath); // Create PerformanceAnalysisResults - runID.html List<string> xmlPaths = new List<string>(); xmlPaths.Add(xmlPath); string analysisPath = Path.Combine(project.OutputDir, "performanceAnalysisResults - " + project.OutputBaseFileName + ".html"); Analysis.AnalysisHelpers.runAnalysis(xmlPaths, project.baselineXML, htmlOutputPath: analysisPath); var failedTests = Analysis.AnalysisHelpers.getFailedTests(analysisPath); if(failedTests.Count != 0) { string trailingS = failedTests.Count == 1 ? string.Empty : "s"; Console.WriteLine($"Performance regressions were found in the following {failedTests.Count} test{trailingS}:"); foreach(var test in failedTests) { Console.WriteLine($" {test}"); } } // Prepare ngen symbols for zipping string srcNGENPath = Path.Combine(project.OutputDir, project.OutputBaseFileName + ".etl" + ".NGENPDB"); string destNGENPath = Path.Combine(project.OutputDir, "symbols"); if (Directory.Exists(destNGENPath)) Directory.Delete(destNGENPath, recursive:true); if (Directory.Exists(srcNGENPath)) Directory.Move(srcNGENPath, destNGENPath); // Zip files to runID.etl.zip string[] filesToZip = new string[] { xmlPath, scenarioRangesPath, tempResultsPath, analysisPath, project.EtlPath }; foreach(var file in filesToZip) { Stopwatch sw = new Stopwatch(); // poor man's Thread.Sleep() sw.Start(); while(IsFileLocked(file)) { for(int i=0; ; i++) { if(i % 500 == 0) { sw.Stop(); if (sw.ElapsedMilliseconds > 100) break; else sw.Start(); } } } } // Zipping does not work for xunit.performance.run.core. Replaced with noops ZipperWrapper zipper = new ZipperWrapper(zipPath, filesToZip); if (Directory.Exists(destNGENPath)) zipper.QueueAddFileOrDir(destNGENPath); zipper.CloseZipFile(); File.Delete(project.EtlPath); File.Delete(scenarioRangesPath); } }
private void RunTests(XunitPerformanceProject project) { string xmlPath = Path.Combine(project.OutputDir, project.OutputBaseFileName + ".xml"); var commandLineArgs = new StringBuilder(); if (!string.IsNullOrEmpty(project.RunnerHost)) { commandLineArgs.Append("\""); commandLineArgs.Append(project.RunnerCommand); commandLineArgs.Append("\" "); } foreach (var assembly in project.Assemblies) { commandLineArgs.Append("\""); commandLineArgs.Append(assembly.AssemblyFilename); commandLineArgs.Append("\" "); } foreach (var testClass in project.Filters.IncludedClasses) { commandLineArgs.Append("-class "); commandLineArgs.Append(testClass); commandLineArgs.Append(" "); } foreach (var testMethod in project.Filters.IncludedMethods) { commandLineArgs.Append("-method "); commandLineArgs.Append(testMethod); commandLineArgs.Append(" "); } foreach (var trait in project.Filters.IncludedTraits) { foreach (var traitValue in trait.Value) { commandLineArgs.Append("-trait \""); commandLineArgs.Append(trait.Key); commandLineArgs.Append("="); commandLineArgs.Append(traitValue); commandLineArgs.Append("\" "); } } foreach (var trait in project.Filters.ExcludedTraits) { foreach (var traitValue in trait.Value) { commandLineArgs.Append("-notrait \""); commandLineArgs.Append(trait.Key); commandLineArgs.Append("="); commandLineArgs.Append(traitValue); commandLineArgs.Append("\" "); } } if (!string.IsNullOrEmpty(project.RunnerArgs)) { commandLineArgs.Append(project.RunnerArgs); commandLineArgs.Append(" "); } commandLineArgs.Append("-xml \""); commandLineArgs.Append(xmlPath); commandLineArgs.Append("\""); ProcessStartInfo startInfo = new ProcessStartInfo() { FileName = project.RunnerHost ?? project.RunnerCommand, Arguments = commandLineArgs.ToString(), UseShellExecute = false, }; startInfo.Environment["XUNIT_PERFORMANCE_RUN_ID"] = project.RunId; startInfo.Environment["XUNIT_PERFORMANCE_MIN_ITERATION"] = "10"; startInfo.Environment["XUNIT_PERFORMANCE_MAX_ITERATION"] = "1000"; startInfo.Environment["XUNIT_PERFORMANCE_MAX_TOTAL_MILLISECONDS"] = "1000"; startInfo.Environment["COMPLUS_gcConcurrent"] = "0"; startInfo.Environment["COMPLUS_gcServer"] = "0"; var logger = GetPerformanceMetricLogger(project); using (logger.StartLogging(startInfo)) { PrintIfVerbose($@"Launching runner: Runner: {startInfo.FileName} Arguments: {startInfo.Arguments}"); try { using (var proc = Process.Start(startInfo)) { proc.EnableRaisingEvents = true; proc.WaitForExit(); } } catch (Exception ex) { throw new Exception($"Could not launch the test runner, {startInfo.FileName}", innerException: ex); } } using (var evaluationContext = logger.GetReader()) { var xmlDoc = XDocument.Load(xmlPath); foreach (var testElem in xmlDoc.Descendants("test")) { var testName = testElem.Attribute("name").Value; var perfElem = new XElement("performance", new XAttribute("runid", project.RunId), new XAttribute("etl", Path.GetFullPath(evaluationContext.LogPath))); testElem.Add(perfElem); var metrics = evaluationContext.GetMetrics(testName); if (metrics != null) { var metricsElem = new XElement("metrics"); perfElem.Add(metricsElem); foreach (var metric in metrics) { metricsElem.Add(new XElement(metric.Id, new XAttribute("displayName", metric.DisplayName), new XAttribute("unit", metric.Unit))); } } var iterations = evaluationContext.GetValues(testName); if (iterations != null) { var iterationsElem = new XElement("iterations"); perfElem.Add(iterationsElem); for (int i = 0; i < iterations.Count; i++) { var iteration = iterations[i]; if (iteration != null) { var iterationElem = new XElement("iteration", new XAttribute("index", i)); iterationsElem.Add(iterationElem); foreach (var value in iteration) { iterationElem.Add(new XAttribute(value.Key, value.Value.ToString("R"))); } } } } } using (var xmlFile = File.Create(xmlPath)) xmlDoc.Save(xmlFile); } }
protected override IPerformanceMetricLogger GetPerformanceMetricLogger(XunitPerformanceProject project) { return(new EtwPerformanceMetricLogger(project, this)); }
protected override IPerformanceMetricLogger GetPerformanceMetricLogger(XunitPerformanceProject project) { return new CSVMetricLogger(project); }