public void AddBaselinePerformanceTestRunResults(
     TestResultXmlParser testResultXmlParser,
     List <PerformanceTestRunResult> baselinePerformanceTestRunResults,
     List <TestResult> baselineTestResults)
 {
     AddTestResults(testResultXmlParser, baselinePerformanceTestRunResults, baselineTestResults, baselineTestResults, BaselineXmlDirectoryPaths, BaselineXmlFilePaths, true);
 }
 public void AddPerformanceTestRunResults(
     TestResultXmlParser testResultXmlParser,
     List <PerformanceTestRunResult> performanceTestRunResults,
     List <TestResult> testResults,
     List <TestResult> baselineTestResults)
 {
     AddTestResults(testResultXmlParser, performanceTestRunResults, testResults, baselineTestResults, ResultXmlDirectoryPaths, ResultXmlFilePaths);
 }
Example #3
0
        private static void Main(string[] args)
        {
            var aggregateTestRunResults           = new List <PerformanceTestRunResult>();
            var baselinePerformanceTestRunResults = new List <PerformanceTestRunResult>();
            var baselineTestResults       = new List <TestResult>();
            var performanceTestRunResults = new List <PerformanceTestRunResult>();
            var testResults          = new List <TestResult>();
            var performanceBenchmark = new PerformanceBenchmark(ExcludedConfigFieldNames);
            var optionsParser        = new OptionsParser();

            optionsParser.ParseOptions(performanceBenchmark, args);
            var testResultXmlParser = new TestResultXmlParser();

            if (performanceBenchmark.BaselineResultFilesExist)
            {
                performanceBenchmark.AddBaselinePerformanceTestRunResults(testResultXmlParser, baselinePerformanceTestRunResults, baselineTestResults);

                if (baselinePerformanceTestRunResults.Any())
                {
                    aggregateTestRunResults.AddRange(baselinePerformanceTestRunResults);
                }
                else
                {
                    Environment.Exit(1);
                }
            }

            if (performanceBenchmark.ResultFilesExist)
            {
                performanceBenchmark.AddPerformanceTestRunResults(testResultXmlParser, performanceTestRunResults, testResults, baselineTestResults);

                if (performanceTestRunResults.Any())
                {
                    aggregateTestRunResults.AddRange(performanceTestRunResults);
                }
                else
                {
                    Environment.Exit(1);
                }
            }

            var reportWriter = new ReportWriter(ExcludedConfigFieldNames);

            reportWriter.WriteReport(
                aggregateTestRunResults,
                performanceBenchmark.MetadataValidator,
                performanceBenchmark.SigFig,
                performanceBenchmark.ReportDirPath,
                performanceBenchmark.BaselineResultFilesExist);
        }
Example #4
0
        private static void Main(string[] args)
        {
            var aggregateTestRunResults           = new List <PerformanceTestRunResult>();
            var baselinePerformanceTestRunResults = new List <PerformanceTestRunResult>();
            var baselineTestResults       = new List <TestResult>();
            var performanceTestRunResults = new List <PerformanceTestRunResult>();
            var testResults          = new List <TestResult>();
            var performanceBenchmark = new PerformanceBenchmark(ExcludedConfigFieldNames);
            var optionsParser        = new OptionsParser();

            optionsParser.ParseOptions(performanceBenchmark, args);
            var testResultXmlParser = new TestResultXmlParser();

            if (performanceBenchmark.BaselineResultFilesExist)
            {
                performanceBenchmark.AddBaselinePerformanceTestRunResults(testResultXmlParser, baselinePerformanceTestRunResults, baselineTestResults);

                if (baselinePerformanceTestRunResults.Any())
                {
                    aggregateTestRunResults.AddRange(baselinePerformanceTestRunResults);
                }
                else
                {
                    Environment.Exit(1);
                }
            }

            if (performanceBenchmark.ResultFilesExist)
            {
                performanceBenchmark.AddPerformanceTestRunResults(testResultXmlParser, performanceTestRunResults, testResults, baselineTestResults);

                if (performanceTestRunResults.Any())
                {
                    aggregateTestRunResults.AddRange(performanceTestRunResults);
                }
                else
                {
                    Environment.Exit(1);
                }
            }

            var performanceTestResults = new PerformanceTestRunResult[0];

            if (aggregateTestRunResults.Any(a => a.IsBaseline))
            {
                Array.Resize(ref performanceTestResults, 1);
                performanceTestResults[0] = aggregateTestRunResults.First(a => a.IsBaseline);
            }

            var nonBaselineTestRunResults = aggregateTestRunResults.Where(a => !a.IsBaseline).ToList();

            nonBaselineTestRunResults.Sort((run1, run2) => string.Compare(run1.ResultName, run2.ResultName, StringComparison.Ordinal));


            foreach (var performanceTestRunResult in nonBaselineTestRunResults)
            {
                Array.Resize(ref performanceTestResults, performanceTestResults.Length + 1);
                performanceTestResults[performanceTestResults.Length - 1] = performanceTestRunResult;
            }


            var reportWriter = new ReportWriter(performanceBenchmark.TestRunMetadataProcessor);

            reportWriter.WriteReport(
                performanceTestResults,
                performanceBenchmark.SigFig,
                performanceBenchmark.ReportDirPath,
                performanceBenchmark.ReportHtmlName,
                performanceBenchmark.BaselineResultFilesExist);
        }
Example #5
0
        private void AddTestResults(
            TestResultXmlParser testResultXmlParser,
            List <PerformanceTestRunResult> testRunResults,
            List <TestResult> testResults,
            List <TestResult> baselineTestResults,
            HashSet <string> xmlDirectoryPaths,
            HashSet <string> xmlFileNamePaths,
            bool isBaseline = false)
        {
            if (!isBaseline && xmlDirectoryPaths != null && xmlDirectoryPaths.Any())
            {
                foreach (var xmlDirectory in xmlDirectoryPaths)
                {
                    var xmlFileNames = GetAllXmlFileNames(xmlDirectory);

                    foreach (var xmlFileName in xmlFileNames)
                    {
                        xmlFileNamePaths.Add(xmlFileName);
                    }
                }
            }

            if (xmlFileNamePaths.Any())
            {
                var perfTestRuns = new List <KeyValuePair <string, PerformanceTestRun> >();

                foreach (var xmlFileNamePath in xmlFileNamePaths)
                {
                    var performanceTestRun = testResultXmlParser.GetPerformanceTestRunFromXml(xmlFileNamePath);
                    if (performanceTestRun != null && performanceTestRun.Results.Any())
                    {
                        perfTestRuns.Add(
                            new KeyValuePair <string, PerformanceTestRun>(xmlFileNamePath, performanceTestRun));
                    }
                }

                perfTestRuns.Sort((run1, run2) => string.Compare(run1.Key, run2.Key, StringComparison.Ordinal));
                var resultFilesOrderedByResultName = perfTestRuns.ToArray();

                for (var i = 0; i < resultFilesOrderedByResultName.Length; i++)
                {
                    var performanceTestRun =
                        testResultXmlParser.GetPerformanceTestRunFromXml(resultFilesOrderedByResultName[i].Key);

                    if (performanceTestRun != null && performanceTestRun.Results.Any())
                    {
                        var results = performanceTestRunProcessor.GetTestResults(performanceTestRun);
                        if (!results.Any())
                        {
                            Console.ForegroundColor = ConsoleColor.Yellow;
                            Console.WriteLine("No performance test data found to report in: {0}",
                                              resultFilesOrderedByResultName[i].Key);
                            Console.ResetColor();
                            continue;
                        }

                        testResults.AddRange(results);

                        performanceTestRunProcessor.UpdateTestResultsBasedOnBaselineResults(baselineTestResults,
                                                                                            testResults, SigFig);

                        TestRunMetadataProcessor.ProcessMetadata(performanceTestRun,
                                                                 resultFilesOrderedByResultName[i].Key);

                        testRunResults.Add(performanceTestRunProcessor.CreateTestRunResult
                                           (
                                               performanceTestRun,
                                               results,
                                               Path.GetFileNameWithoutExtension(resultFilesOrderedByResultName[i].Key),
                                               isBaseline)
                                           );
                    }
                }
            }
        }
Example #6
0
        private static int Main(string[] args)
        {
            var aggregateTestRunResults           = new List <PerformanceTestRunResult>();
            var baselinePerformanceTestRunResults = new List <PerformanceTestRunResult>();
            var baselineTestResults       = new List <TestResult>();
            var performanceTestRunResults = new List <PerformanceTestRunResult>();
            var testResults          = new List <TestResult>();
            var performanceBenchmark = new PerformanceBenchmark(ExcludedConfigFieldNames);
            var optionsParser        = new OptionsParser();

            optionsParser.ParseOptions(performanceBenchmark, args);
            var testResultXmlParser = new TestResultXmlParser();

            if (performanceBenchmark.BaselineResultFilesExist)
            {
                performanceBenchmark.AddBaselinePerformanceTestRunResults(testResultXmlParser, baselinePerformanceTestRunResults, baselineTestResults);

                if (baselinePerformanceTestRunResults.Any())
                {
                    aggregateTestRunResults.AddRange(baselinePerformanceTestRunResults);
                }
                else
                {
                    Environment.Exit(1);
                }
            }

            if (performanceBenchmark.ResultFilesExist)
            {
                performanceBenchmark.AddPerformanceTestRunResults(testResultXmlParser, performanceTestRunResults, testResults, baselineTestResults);

                if (performanceTestRunResults.Any())
                {
                    aggregateTestRunResults.AddRange(performanceTestRunResults);
                }
                else
                {
                    Environment.Exit(1);
                }
            }

            var performanceTestResults = new PerformanceTestRunResult[0];

            // If we have a baseline
            if (aggregateTestRunResults.Any(a => a.IsBaseline))
            {
                // Insert the baseline in the front of the array results; this way we can display the baseline first in the report
                Array.Resize(ref performanceTestResults, 1);
                performanceTestResults[0] = aggregateTestRunResults.First(a => a.IsBaseline);
            }

            var nonBaselineTestRunResults = aggregateTestRunResults.Where(a => !a.IsBaseline).ToList();

            nonBaselineTestRunResults.Sort((run1, run2) => string.Compare(run1.ResultName, run2.ResultName, StringComparison.Ordinal));


            foreach (var performanceTestRunResult in nonBaselineTestRunResults)
            {
                Array.Resize(ref performanceTestResults, performanceTestResults.Length + 1);
                performanceTestResults[performanceTestResults.Length - 1] = performanceTestRunResult;
            }

            performanceBenchmark.TestRunMetadataProcessor.PerformFinalMetadataUpdate(performanceBenchmark);

            var reportWriter = new ReportWriter(performanceBenchmark.TestRunMetadataProcessor);

            reportWriter.WriteReport(
                performanceTestResults,
                performanceBenchmark.SigFig,
                performanceBenchmark.ReportDirPath,
                performanceBenchmark.BaselineResultFilesExist);

            return(WriteFailedTestsAndMetricsToConsole(performanceTestResults, performanceBenchmark));
        }