public void AddBaselinePerformanceTestRunResults( TestResultXmlParser testResultXmlParser, List <PerformanceTestRunResult> baselinePerformanceTestRunResults, List <TestResult> baselineTestResults) { AddTestResults(testResultXmlParser, baselinePerformanceTestRunResults, baselineTestResults, baselineTestResults, BaselineXmlDirectoryPaths, BaselineXmlFilePaths, true); }
public void AddPerformanceTestRunResults( TestResultXmlParser testResultXmlParser, List <PerformanceTestRunResult> performanceTestRunResults, List <TestResult> testResults, List <TestResult> baselineTestResults) { AddTestResults(testResultXmlParser, performanceTestRunResults, testResults, baselineTestResults, ResultXmlDirectoryPaths, ResultXmlFilePaths); }
private static void Main(string[] args) { var aggregateTestRunResults = new List <PerformanceTestRunResult>(); var baselinePerformanceTestRunResults = new List <PerformanceTestRunResult>(); var baselineTestResults = new List <TestResult>(); var performanceTestRunResults = new List <PerformanceTestRunResult>(); var testResults = new List <TestResult>(); var performanceBenchmark = new PerformanceBenchmark(ExcludedConfigFieldNames); var optionsParser = new OptionsParser(); optionsParser.ParseOptions(performanceBenchmark, args); var testResultXmlParser = new TestResultXmlParser(); if (performanceBenchmark.BaselineResultFilesExist) { performanceBenchmark.AddBaselinePerformanceTestRunResults(testResultXmlParser, baselinePerformanceTestRunResults, baselineTestResults); if (baselinePerformanceTestRunResults.Any()) { aggregateTestRunResults.AddRange(baselinePerformanceTestRunResults); } else { Environment.Exit(1); } } if (performanceBenchmark.ResultFilesExist) { performanceBenchmark.AddPerformanceTestRunResults(testResultXmlParser, performanceTestRunResults, testResults, baselineTestResults); if (performanceTestRunResults.Any()) { aggregateTestRunResults.AddRange(performanceTestRunResults); } else { Environment.Exit(1); } } var reportWriter = new ReportWriter(ExcludedConfigFieldNames); reportWriter.WriteReport( aggregateTestRunResults, performanceBenchmark.MetadataValidator, performanceBenchmark.SigFig, performanceBenchmark.ReportDirPath, performanceBenchmark.BaselineResultFilesExist); }
private static void Main(string[] args) { var aggregateTestRunResults = new List <PerformanceTestRunResult>(); var baselinePerformanceTestRunResults = new List <PerformanceTestRunResult>(); var baselineTestResults = new List <TestResult>(); var performanceTestRunResults = new List <PerformanceTestRunResult>(); var testResults = new List <TestResult>(); var performanceBenchmark = new PerformanceBenchmark(ExcludedConfigFieldNames); var optionsParser = new OptionsParser(); optionsParser.ParseOptions(performanceBenchmark, args); var testResultXmlParser = new TestResultXmlParser(); if (performanceBenchmark.BaselineResultFilesExist) { performanceBenchmark.AddBaselinePerformanceTestRunResults(testResultXmlParser, baselinePerformanceTestRunResults, baselineTestResults); if (baselinePerformanceTestRunResults.Any()) { aggregateTestRunResults.AddRange(baselinePerformanceTestRunResults); } else { Environment.Exit(1); } } if (performanceBenchmark.ResultFilesExist) { performanceBenchmark.AddPerformanceTestRunResults(testResultXmlParser, performanceTestRunResults, testResults, baselineTestResults); if (performanceTestRunResults.Any()) { aggregateTestRunResults.AddRange(performanceTestRunResults); } else { Environment.Exit(1); } } var performanceTestResults = new PerformanceTestRunResult[0]; if (aggregateTestRunResults.Any(a => a.IsBaseline)) { Array.Resize(ref performanceTestResults, 1); performanceTestResults[0] = aggregateTestRunResults.First(a => a.IsBaseline); } var nonBaselineTestRunResults = aggregateTestRunResults.Where(a => !a.IsBaseline).ToList(); nonBaselineTestRunResults.Sort((run1, run2) => string.Compare(run1.ResultName, run2.ResultName, StringComparison.Ordinal)); foreach (var performanceTestRunResult in nonBaselineTestRunResults) { Array.Resize(ref performanceTestResults, performanceTestResults.Length + 1); performanceTestResults[performanceTestResults.Length - 1] = performanceTestRunResult; } var reportWriter = new ReportWriter(performanceBenchmark.TestRunMetadataProcessor); reportWriter.WriteReport( performanceTestResults, performanceBenchmark.SigFig, performanceBenchmark.ReportDirPath, performanceBenchmark.ReportHtmlName, performanceBenchmark.BaselineResultFilesExist); }
private void AddTestResults( TestResultXmlParser testResultXmlParser, List <PerformanceTestRunResult> testRunResults, List <TestResult> testResults, List <TestResult> baselineTestResults, HashSet <string> xmlDirectoryPaths, HashSet <string> xmlFileNamePaths, bool isBaseline = false) { if (!isBaseline && xmlDirectoryPaths != null && xmlDirectoryPaths.Any()) { foreach (var xmlDirectory in xmlDirectoryPaths) { var xmlFileNames = GetAllXmlFileNames(xmlDirectory); foreach (var xmlFileName in xmlFileNames) { xmlFileNamePaths.Add(xmlFileName); } } } if (xmlFileNamePaths.Any()) { var perfTestRuns = new List <KeyValuePair <string, PerformanceTestRun> >(); foreach (var xmlFileNamePath in xmlFileNamePaths) { var performanceTestRun = testResultXmlParser.GetPerformanceTestRunFromXml(xmlFileNamePath); if (performanceTestRun != null && performanceTestRun.Results.Any()) { perfTestRuns.Add( new KeyValuePair <string, PerformanceTestRun>(xmlFileNamePath, performanceTestRun)); } } perfTestRuns.Sort((run1, run2) => string.Compare(run1.Key, run2.Key, StringComparison.Ordinal)); var resultFilesOrderedByResultName = perfTestRuns.ToArray(); for (var i = 0; i < resultFilesOrderedByResultName.Length; i++) { var performanceTestRun = testResultXmlParser.GetPerformanceTestRunFromXml(resultFilesOrderedByResultName[i].Key); if (performanceTestRun != null && performanceTestRun.Results.Any()) { var results = performanceTestRunProcessor.GetTestResults(performanceTestRun); if (!results.Any()) { Console.ForegroundColor = ConsoleColor.Yellow; Console.WriteLine("No performance test data found to report in: {0}", resultFilesOrderedByResultName[i].Key); Console.ResetColor(); continue; } testResults.AddRange(results); performanceTestRunProcessor.UpdateTestResultsBasedOnBaselineResults(baselineTestResults, testResults, SigFig); TestRunMetadataProcessor.ProcessMetadata(performanceTestRun, resultFilesOrderedByResultName[i].Key); testRunResults.Add(performanceTestRunProcessor.CreateTestRunResult ( performanceTestRun, results, Path.GetFileNameWithoutExtension(resultFilesOrderedByResultName[i].Key), isBaseline) ); } } } }
private static int Main(string[] args) { var aggregateTestRunResults = new List <PerformanceTestRunResult>(); var baselinePerformanceTestRunResults = new List <PerformanceTestRunResult>(); var baselineTestResults = new List <TestResult>(); var performanceTestRunResults = new List <PerformanceTestRunResult>(); var testResults = new List <TestResult>(); var performanceBenchmark = new PerformanceBenchmark(ExcludedConfigFieldNames); var optionsParser = new OptionsParser(); optionsParser.ParseOptions(performanceBenchmark, args); var testResultXmlParser = new TestResultXmlParser(); if (performanceBenchmark.BaselineResultFilesExist) { performanceBenchmark.AddBaselinePerformanceTestRunResults(testResultXmlParser, baselinePerformanceTestRunResults, baselineTestResults); if (baselinePerformanceTestRunResults.Any()) { aggregateTestRunResults.AddRange(baselinePerformanceTestRunResults); } else { Environment.Exit(1); } } if (performanceBenchmark.ResultFilesExist) { performanceBenchmark.AddPerformanceTestRunResults(testResultXmlParser, performanceTestRunResults, testResults, baselineTestResults); if (performanceTestRunResults.Any()) { aggregateTestRunResults.AddRange(performanceTestRunResults); } else { Environment.Exit(1); } } var performanceTestResults = new PerformanceTestRunResult[0]; // If we have a baseline if (aggregateTestRunResults.Any(a => a.IsBaseline)) { // Insert the baseline in the front of the array results; this way we can display the baseline first in the report Array.Resize(ref performanceTestResults, 1); performanceTestResults[0] = aggregateTestRunResults.First(a => a.IsBaseline); } var nonBaselineTestRunResults = aggregateTestRunResults.Where(a => !a.IsBaseline).ToList(); nonBaselineTestRunResults.Sort((run1, run2) => string.Compare(run1.ResultName, run2.ResultName, StringComparison.Ordinal)); foreach (var performanceTestRunResult in nonBaselineTestRunResults) { Array.Resize(ref performanceTestResults, performanceTestResults.Length + 1); performanceTestResults[performanceTestResults.Length - 1] = performanceTestRunResult; } performanceBenchmark.TestRunMetadataProcessor.PerformFinalMetadataUpdate(performanceBenchmark); var reportWriter = new ReportWriter(performanceBenchmark.TestRunMetadataProcessor); reportWriter.WriteReport( performanceTestResults, performanceBenchmark.SigFig, performanceBenchmark.ReportDirPath, performanceBenchmark.BaselineResultFilesExist); return(WriteFailedTestsAndMetricsToConsole(performanceTestResults, performanceBenchmark)); }