public void ParseOptions(PerformanceBenchmark performanceBenchmark, IEnumerable <string> args)
{
var os = GetOptions(performanceBenchmark);
try
{
var remaining = os.Parse(args);
if (help)
{
ShowHelp(string.Empty, os);
}
if (!performanceBenchmark.ResultXmlFilePaths.Any() && !performanceBenchmark.ResultXmlDirectoryPaths.Any())
{
ShowHelp("Missing required option --results=(filePath|directoryPath)", os);
}
if (remaining.Any())
{
var errorMessage = string.Format("Unknown option: '{0}.\r\n'", remaining[0]);
ShowHelp(errorMessage, os);
}
}
catch (Exception e)
{
ShowHelp(string.Format("Error encountered while parsing option: {0}.\r\n", e.Message), os);
}
}
public void PerformFinalMetadataUpdate(PerformanceBenchmark performanceBenchmark)
{
// The keys in the extraMetadataExtractFields structure have additional embedded metadata that we extract out.
// This renders the raw, unextracted, value of this field unusable, so we discard it
foreach (var metadataName in extraMetadataExtractFields.Keys)
{
foreach (var typeMetadata in performanceBenchmark.TestRunMetadataProcessor.TypeMetadata)
{
typeMetadata.FieldGroups.RemoveAll(fg => fg.FieldName.Equals(metadataName));
}
}
}
private static void Main(string[] args)
{
var aggregateTestRunResults = new List <PerformanceTestRunResult>();
var baselinePerformanceTestRunResults = new List <PerformanceTestRunResult>();
var baselineTestResults = new List <TestResult>();
var performanceTestRunResults = new List <PerformanceTestRunResult>();
var testResults = new List <TestResult>();
var performanceBenchmark = new PerformanceBenchmark(ExcludedConfigFieldNames);
var optionsParser = new OptionsParser();
optionsParser.ParseOptions(performanceBenchmark, args);
var testResultXmlParser = new TestResultXmlParser();
if (performanceBenchmark.BaselineResultFilesExist)
{
performanceBenchmark.AddBaselinePerformanceTestRunResults(testResultXmlParser, baselinePerformanceTestRunResults, baselineTestResults);
if (baselinePerformanceTestRunResults.Any())
{
aggregateTestRunResults.AddRange(baselinePerformanceTestRunResults);
}
else
{
Environment.Exit(1);
}
}
if (performanceBenchmark.ResultFilesExist)
{
performanceBenchmark.AddPerformanceTestRunResults(testResultXmlParser, performanceTestRunResults, testResults, baselineTestResults);
if (performanceTestRunResults.Any())
{
aggregateTestRunResults.AddRange(performanceTestRunResults);
}
else
{
Environment.Exit(1);
}
}
var reportWriter = new ReportWriter(ExcludedConfigFieldNames);
reportWriter.WriteReport(
aggregateTestRunResults,
performanceBenchmark.MetadataValidator,
performanceBenchmark.SigFig,
performanceBenchmark.ReportDirPath,
performanceBenchmark.BaselineResultFilesExist);
}
private OptionSet GetOptions(PerformanceBenchmark performanceBenchmark)
{
return(new OptionSet()
.Add("?|help|h", "Prints out the options.", option => help = option != null)
.Add("results|testresultsxmlsource=", "REQUIRED - Path to a test result XML filename OR directory. Directories are searched resursively. You can repeat this option with multiple result file or directory paths.",
xmlsource =>
{
performanceBenchmark.AddXmlSourcePath(xmlsource, "results", ResultType.Test);
})
.Add("baseline|baselinexmlsource:", "OPTIONAL - Path to a baseline XML filename.",
xmlsource =>
{
performanceBenchmark.AddXmlSourcePath(xmlsource, "baseline", ResultType.Baseline);
})
.Add("report|reportdirpath:", "OPTIONAL - Path to where the report will be written. Default is current working directory.",
performanceBenchmark.AddReportDirPath));
}
private OptionSet GetOptions(PerformanceBenchmark performanceBenchmark)
{
var optionsSet = new OptionSet();
optionsSet.Add("?|help|h", "Prints out the options.", option => help = option != null);
optionsSet.Add(
"results|testresultsxmlsource=",
"REQUIRED - Path to a test result XML filename OR directory. Directories are searched resursively. You can repeat this option with multiple result file or directory paths.",
xmlsource => performanceBenchmark.AddXmlSourcePath(xmlsource, "results", ResultType.Test));
optionsSet.Add(
"baseline|baselinexmlsource:", "OPTIONAL - Path to a baseline XML filename.",
xmlsource => performanceBenchmark.AddXmlSourcePath(xmlsource, "baseline", ResultType.Baseline));
optionsSet.Add(
"report|reportdirpath:", "OPTIONAL - Path to where the report will be written. Default is current working directory.",
performanceBenchmark.AddReportDirPath);
optionsSet.Add("failonbaseline",
"Enable return '1' by the reporter if a baseline is passed in and one or more matching configs is out of threshold. Disabled is default. Use option to enable, or use option and append '-' to explicitly disable.",
option => performanceBenchmark.FailOnBaseline = option != null);
return(optionsSet);
}
private OptionSet GetOptions(PerformanceBenchmark performanceBenchmark)
{
var optionsSet = new OptionSet();
optionsSet.Add("?|help|h", "Prints out the options.", option => help = option != null);
optionsSet.Add("dataversion|version=", "Sets Expected Perf Data Version for Results and Baseline Files (1 = V1 2 = V2). Versions of Unity Perf Framework 2.0 or newer will use the V2 data format. If no arg is provided we assume the format is V2", version => performanceBenchmark.SetDataVersion(version));
optionsSet.Add("fileformat|format=", "Sets Expected File Format for Results and Baseline Files. If no arg is provided we assume the format is XML", filtype => performanceBenchmark.SetFileType(filtype));
optionsSet.Add(
"results|testresultsxmlsource=",
"REQUIRED - Path to a test result XML filename OR directory. Directories are searched resursively. You can repeat this option with multiple result file or directory paths.",
xmlsource => performanceBenchmark.AddSourcePath(xmlsource, "results", ResultType.Test));
optionsSet.Add(
"baseline|baselinexmlsource:", "OPTIONAL - Path to a baseline XML filename.",
xmlsource => performanceBenchmark.AddSourcePath(xmlsource, "baseline", ResultType.Baseline));
optionsSet.Add(
"report|reportdirpath:", "OPTIONAL - Path to where the report will be written. Default is current working directory.",
performanceBenchmark.AddReportDirPath);
optionsSet.Add("failonbaseline",
"Enable return '1' by the reporter if a baseline is passed in and one or more matching configs is out of threshold. Disabled is default. Use option to enable, or use option and append '-' to explicitly disable.",
option => performanceBenchmark.FailOnBaseline = option != null);
return(optionsSet);
}
private static void Main(string[] args)
{
var aggregateTestRunResults = new List <PerformanceTestRunResult>();
var baselinePerformanceTestRunResults = new List <PerformanceTestRunResult>();
var baselineTestResults = new List <TestResult>();
var performanceTestRunResults = new List <PerformanceTestRunResult>();
var testResults = new List <TestResult>();
var performanceBenchmark = new PerformanceBenchmark(ExcludedConfigFieldNames);
var optionsParser = new OptionsParser();
optionsParser.ParseOptions(performanceBenchmark, args);
var testResultXmlParser = new TestResultXmlParser();
if (performanceBenchmark.BaselineResultFilesExist)
{
performanceBenchmark.AddBaselinePerformanceTestRunResults(testResultXmlParser, baselinePerformanceTestRunResults, baselineTestResults);
if (baselinePerformanceTestRunResults.Any())
{
aggregateTestRunResults.AddRange(baselinePerformanceTestRunResults);
}
else
{
Environment.Exit(1);
}
}
if (performanceBenchmark.ResultFilesExist)
{
performanceBenchmark.AddPerformanceTestRunResults(testResultXmlParser, performanceTestRunResults, testResults, baselineTestResults);
if (performanceTestRunResults.Any())
{
aggregateTestRunResults.AddRange(performanceTestRunResults);
}
else
{
Environment.Exit(1);
}
}
var performanceTestResults = new PerformanceTestRunResult[0];
if (aggregateTestRunResults.Any(a => a.IsBaseline))
{
Array.Resize(ref performanceTestResults, 1);
performanceTestResults[0] = aggregateTestRunResults.First(a => a.IsBaseline);
}
var nonBaselineTestRunResults = aggregateTestRunResults.Where(a => !a.IsBaseline).ToList();
nonBaselineTestRunResults.Sort((run1, run2) => string.Compare(run1.ResultName, run2.ResultName, StringComparison.Ordinal));
foreach (var performanceTestRunResult in nonBaselineTestRunResults)
{
Array.Resize(ref performanceTestResults, performanceTestResults.Length + 1);
performanceTestResults[performanceTestResults.Length - 1] = performanceTestRunResult;
}
var reportWriter = new ReportWriter(performanceBenchmark.TestRunMetadataProcessor);
reportWriter.WriteReport(
performanceTestResults,
performanceBenchmark.SigFig,
performanceBenchmark.ReportDirPath,
performanceBenchmark.ReportHtmlName,
performanceBenchmark.BaselineResultFilesExist);
}
private static int Main(string[] args)
{
var aggregateTestRunResults = new List <PerformanceTestRunResult>();
var baselinePerformanceTestRunResults = new List <PerformanceTestRunResult>();
var baselineTestResults = new List <TestResult>();
var performanceTestRunResults = new List <PerformanceTestRunResult>();
var testResults = new List <TestResult>();
var performanceBenchmark = new PerformanceBenchmark(ExcludedConfigFieldNames);
var optionsParser = new OptionsParser();
optionsParser.ParseOptions(performanceBenchmark, args);
var testResultXmlParser = new TestResultXmlParser();
if (performanceBenchmark.BaselineResultFilesExist)
{
performanceBenchmark.AddBaselinePerformanceTestRunResults(testResultXmlParser, baselinePerformanceTestRunResults, baselineTestResults);
if (baselinePerformanceTestRunResults.Any())
{
aggregateTestRunResults.AddRange(baselinePerformanceTestRunResults);
}
else
{
Environment.Exit(1);
}
}
if (performanceBenchmark.ResultFilesExist)
{
performanceBenchmark.AddPerformanceTestRunResults(testResultXmlParser, performanceTestRunResults, testResults, baselineTestResults);
if (performanceTestRunResults.Any())
{
aggregateTestRunResults.AddRange(performanceTestRunResults);
}
else
{
Environment.Exit(1);
}
}
var performanceTestResults = new PerformanceTestRunResult[0];
// If we have a baseline
if (aggregateTestRunResults.Any(a => a.IsBaseline))
{
// Insert the baseline in the front of the array results; this way we can display the baseline first in the report
Array.Resize(ref performanceTestResults, 1);
performanceTestResults[0] = aggregateTestRunResults.First(a => a.IsBaseline);
}
var nonBaselineTestRunResults = aggregateTestRunResults.Where(a => !a.IsBaseline).ToList();
nonBaselineTestRunResults.Sort((run1, run2) => string.Compare(run1.ResultName, run2.ResultName, StringComparison.Ordinal));
foreach (var performanceTestRunResult in nonBaselineTestRunResults)
{
Array.Resize(ref performanceTestResults, performanceTestResults.Length + 1);
performanceTestResults[performanceTestResults.Length - 1] = performanceTestRunResult;
}
performanceBenchmark.TestRunMetadataProcessor.PerformFinalMetadataUpdate(performanceBenchmark);
var reportWriter = new ReportWriter(performanceBenchmark.TestRunMetadataProcessor);
reportWriter.WriteReport(
performanceTestResults,
performanceBenchmark.SigFig,
performanceBenchmark.ReportDirPath,
performanceBenchmark.BaselineResultFilesExist);
return(WriteFailedTestsAndMetricsToConsole(performanceTestResults, performanceBenchmark));
}
private static int WriteFailedTestsAndMetricsToConsole(PerformanceTestRunResult[] performanceTestResults, PerformanceBenchmark performanceBenchmark)
{
var failedTestsExist = performanceTestResults.SelectMany(ptr => ptr.TestResults)
.Any(tr => tr.State == (int)TestState.Failure);
if (failedTestsExist)
{
WriteLine("FAILURE: One ore more performance test metric aggregations is out of threshold from the baseline value.");
WriteLine("-------------------------------------");
WriteLine(" Performance tests with failed metrics");
WriteLine("-------------------------------------");
foreach (var performanceTestRunResult in performanceTestResults)
{
var failedTests = performanceTestRunResult.TestResults.Where(tr => tr.State == (int)TestState.Failure);
if (failedTests.Any())
{
foreach (var failedTest in failedTests)
{
++indentLevel;
WriteLine("{0}", failedTest.TestName);
var regressedSgs = failedTest.SampleGroupResults.Where(sgr => sgr.Regressed);
foreach (var sampleGroupResult in regressedSgs)
{
WriteLine("----");
WriteLine("Metric : {0}", sampleGroupResult.SampleGroupName);
WriteLine("Aggregation : {0}", sampleGroupResult.AggregationType);
WriteLine("Failed Value : {0,8:F2}", sampleGroupResult.AggregatedValue);
WriteLine("Baseline Value: {0,8:F2}", sampleGroupResult.BaselineValue);
WriteLine("Threshold % : {0,8:F2}", sampleGroupResult.Threshold);
WriteLine("Actual Diff % : {0,8:F2}", Math.Abs(sampleGroupResult.AggregatedValue - sampleGroupResult.BaselineValue) / sampleGroupResult.BaselineValue);
}
--indentLevel;
WriteLine("\r\n");
}
}
}
}
return(performanceBenchmark.FailOnBaseline && failedTestsExist ? 1 : 0);
}
private static void WriteProgressedTestsAndMetricsToConsole(PerformanceTestRunResult[] performanceTestResults, PerformanceBenchmark performanceBenchmark)
{
bool loggedHeader = false;
var passedTestsExist = performanceTestResults.SelectMany(ptr => ptr.TestResults)
.Any(tr => tr.State == (int)TestState.Success);
if (passedTestsExist)
{
foreach (var performanceTestRunResult in performanceTestResults)
{
var passedTests = performanceTestRunResult.TestResults.Where(tr => tr.State == (int)TestState.Success);
if (passedTests.Any())
{
foreach (var tests in passedTests)
{
if (tests.SampleGroupResults.Any(sgr => sgr.Progressed))
{
if (!loggedHeader)
{
loggedHeader = true;
WriteLine("Info: One ore more performance test metric aggregations is out of threshold from the baseline value.");
WriteLine("-------------------------------------");
WriteLine(" Performance tests with Progressed metrics");
WriteLine("-------------------------------------");
}
++indentLevel;
WriteLine("{0}", tests.TestName);
var progressedSgs = tests.SampleGroupResults.Where(sgr => sgr.Progressed);
foreach (var sampleGroupResult in progressedSgs)
{
WriteLine("----");
WriteLine("Metric : {0}", sampleGroupResult.SampleGroupName);
WriteLine("Aggregation : {0}", sampleGroupResult.AggregationType);
WriteLine("New Value : {0,8:F2}", sampleGroupResult.AggregatedValue);
WriteLine("Baseline Value: {0,8:F2}", sampleGroupResult.BaselineValue);
WriteLine("Threshold % : {0,8:F2}", sampleGroupResult.Threshold);
WriteLine("Actual Diff % : {0,8:F2}", Math.Abs(sampleGroupResult.BaselineValue - sampleGroupResult.AggregatedValue) / sampleGroupResult.BaselineValue);
}
--indentLevel;
WriteLine("\r\n");
}
}
}
}
}
}
