private void WriteBenchmarkIntro(StreamWriter textFile, Summary summary, BenchmarkInfo info)
{
var benchmark = summary.Benchmarks.First(b => b.Target.Type.Name == info.ClassName);
var property = benchmark.Target.Type.GetTypeInfo().DeclaredProperties.FirstOrDefault(p => p.Name == "Description");
var description = (string)property?.GetValue(null) ?? "";
if (info.Parameters != null && info.Parameters.Count > 0)
{
foreach (var parameter in info.Parameters)
{
var replaceName = "{" + parameter.Name + "}";
description = description.Replace(replaceName, parameter.Value?.ToString());
}
}
var link = benchmark.Target.Type.FullName.Replace(nameof(DotNet) + "." + nameof(DependencyInjectionBenchmarks), "");
link = link.Replace('.', '/');
link = CodePath + link + ".cs";
textFile.WriteLine("<div class=\"jumbotron\">");
textFile.WriteLine($"<h1>{info.DisplayName}</h1>");
textFile.WriteLine($"<p>{description} The code can be found <a href=\"{link}\">here</a>.</p>");
textFile.WriteLine("</div>");
}
public void setResult(PredictorInfo predictor, BenchmarkInfo benchmark, BenchmarkStatisticsResult value)
{
int predictorIndex;
int benchmarkIndex;
if (!predictorIndexMap.TryGetValue(predictor, out predictorIndex))
{
throw new Exception("Invalid predictor provided.");
}
if (!benchmarkIndexMap.TryGetValue(benchmark, out benchmarkIndex))
{
throw new Exception("Invalid simulator provided.");
}
if ((value != null) && (resultMatrix[predictorIndex, benchmarkIndex] == null))
{
valuesEntered++;
observableCollections[predictor].addSorted(value);
if (newValueReceived != null)
{
newValueReceived(this, new SimulationResultsDictionaryNewValueReceivedEventArgs(predictorIndex, benchmarkIndex, value, observableCollections[predictor]));
}
}
else
{
throw new NotSupportedException();
}
resultMatrix[predictorIndex, benchmarkIndex] = value;
if ((valuesEntered == numberOfPredictors * numberOfBenchmarks) && (filled != null))
{
filled(this, EventArgs.Empty);
}
}
public void addBenchmark(BenchmarkInfo benchmark)
{
try
{
benchmarkIndexMap.Add(benchmark, numberOfBenchmarks++);
}
catch (Exception e)
{
}
}
public IEnumerable <BenchmarkInfo> GetBenchmarksByIsoCountryCode(String isoCode)
{
IEnumerable <BenchmarkInfo> found;
if (this.benchmarkByCountry.Contains(isoCode))
{
found = this.benchmarkByCountry[isoCode];
}
else
{
found = new BenchmarkInfo[] { };
}
return(found);
}
private void Run(BenchmarkInfo benchmarkInfo)
{
switch (Stage)
{
case StageEnum.Build:
benchmarkInfo.BuildSchema();
break;
case StageEnum.TypicalExecution:
_documentExecuter.ExecuteAsync(o =>
{
o.Schema = benchmarkInfo.Schema;
o.Query = benchmarkInfo.Query;
o.Inputs = benchmarkInfo.InputsString?.ToInputs();
}).GetAwaiter().GetResult();
break;
case StageEnum.Parse:
benchmarkInfo.Parse();
break;
case StageEnum.Convert:
benchmarkInfo.Convert();
break;
case StageEnum.Validate:
benchmarkInfo.Validate();
break;
case StageEnum.DeserializeVars:
benchmarkInfo.DeserializeInputs();
break;
case StageEnum.ParseVariables:
benchmarkInfo.ParseVariables();
break;
case StageEnum.Execute:
benchmarkInfo.Execute();
break;
case StageEnum.Serialize:
benchmarkInfo.Serialize();
break;
default:
throw new InvalidOperationException();
}
}
public BenchmarkStatisticsResult getResult(PredictorInfo predictor, BenchmarkInfo benchmark)
{
int predictorIndex;
int benchmarkIndex;
if (!predictorIndexMap.TryGetValue(predictor, out predictorIndex))
{
throw new Exception("Invalid predictor provided.");
}
if (!benchmarkIndexMap.TryGetValue(benchmark, out benchmarkIndex))
{
throw new Exception("Invalid simulator provided.");
}
return(resultMatrix[predictorIndex, benchmarkIndex]);
}
public void GlobalSetup()
{
Func <ISchema> starWarsSchemaBuilder = () =>
{
var services = new ServiceCollection();
services.AddSingleton <StarWarsData>();
services.AddSingleton <StarWarsQuery>();
services.AddSingleton <StarWarsMutation>();
services.AddSingleton <HumanType>();
services.AddSingleton <HumanInputType>();
services.AddSingleton <DroidType>();
services.AddSingleton <CharacterInterface>();
services.AddSingleton <EpisodeEnum>();
services.AddSingleton <ISchema, StarWarsSchema>();
var provider = services.BuildServiceProvider();
var schema = provider.GetRequiredService <ISchema>();
schema.Initialize();
return(schema);
};
_bIntrospection = new BenchmarkInfo(Queries.Introspection, null, starWarsSchemaBuilder);
_bHero = new BenchmarkInfo(Queries.Hero, null, starWarsSchemaBuilder);
Func <ISchema> variableSchemaBuilder = () =>
{
var services = new ServiceCollection();
services.AddSingleton <VariableBenchmark.MyQueryGraphType>();
services.AddSingleton <VariableBenchmark.MyInputObjectGraphType>();
services.AddSingleton <VariableBenchmark.MySubInputObjectGraphType>();
services.AddSingleton <ISchema, VariableBenchmark.MySchema>();
var provider = services.BuildServiceProvider();
var schema = provider.GetRequiredService <ISchema>();
schema.Initialize();
return(schema);
};
_bVariable = new BenchmarkInfo(Queries.VariablesVariable, Benchmarks.Variables.VariablesVariable, variableSchemaBuilder);
_bLiteral = new BenchmarkInfo(Queries.VariablesLiteral, null, variableSchemaBuilder);
}
private void Run(BenchmarkInfo benchmarkInfo)
{
switch (Stage)
{
case StageEnum.Build:
benchmarkInfo.BuildSchema();
break;
case StageEnum.Parse:
benchmarkInfo.Parse();
break;
case StageEnum.Convert:
benchmarkInfo.Convert();
break;
case StageEnum.Validate:
benchmarkInfo.Validate();
break;
case StageEnum.DeserializeVars:
benchmarkInfo.DeserializeInputs();
break;
case StageEnum.ParseVariables:
benchmarkInfo.ParseVariables();
break;
case StageEnum.Execute:
benchmarkInfo.Execute();
break;
case StageEnum.Serialize:
benchmarkInfo.Serialize();
break;
default:
throw new InvalidOperationException();
}
}
public SimulationInfo(PredictorInfo predictorInfo, string benchmarkName, BenchmarkType benchmarkType)
{
this.predictorInfo = predictorInfo;
this.benchmarkInfo = new BenchmarkInfo(benchmarkName, benchmarkType);
}
public SimulationInfo(PredictorInfo predictorInfo, BenchmarkInfo benchmarkInfo)
{
this.predictorInfo = predictorInfo;
this.benchmarkInfo = benchmarkInfo;
}
private void ExportBenchmarkPage(string path, Summary summary, SummaryInfo calculatedHeaders, BenchmarkInfo info)
{
using (var file = File.Create(Path.Combine(path, info.BenchmarkName.Replace('|', '_') + ".html")))
{
using (var textFile = new StreamWriter(file))
{
textFile.WriteLine("<html>");
textFile.Indent(1);
textFile.WriteLine("<head>");
textFile.Indent(2);
textFile.Write(_headString);
textFile.Indent(2);
textFile.WriteLine("</head>");
textFile.Indent(1);
textFile.WriteLine("<body>");
WriteMenu(summary, calculatedHeaders, textFile);
textFile.Indent(2);
textFile.WriteLine("<div class=\"container-fluid theme-showcase\" role=\"main\">");
WriteBenchmarkIntro(textFile, summary, info);
WriteBenchmarkTableData(textFile, summary, info);
OutputSummaryDetails(summary, textFile);
var environmentSwitches = BuildEnvironmentSwitches(summary);
textFile.Write(_scriptString.Replace("{AppendEnvironments}", environmentSwitches));
textFile.Indent(1);
textFile.WriteLine("</div></body>");
textFile.WriteLine("</html>");
}
}
}
private void WriteBenchmarkTableData(StreamWriter textFile, Summary summary, BenchmarkInfo info)
{
var reports = summary.Reports.Where(r => r.Benchmark.Target.Type.Name == info.ClassName);
if (info.Parameters.Count > 0)
{
reports = reports.Where(r =>
{
foreach (var parameter in info.Parameters)
{
if (!r.Benchmark.Parameters.Items.Any(
p => p.Name == parameter.Name && p.Value == parameter.Value))
{
return(false);
}
}
return(true);
});
}
textFile.Indent(4);
textFile.WriteLine("<table class=\"table table-striped table-bordered responsive nowrap\" id=\"benchmarkDataTable\" width=\"100%\" cellspacing=\"0\">");
textFile.WriteLine("<thead>");
textFile.WriteLine("<tr>");
textFile.WriteLine("<th>Container</th>");
textFile.WriteLine("<th>Env</th>");
textFile.WriteLine("<th>Mean (ns)</th>");
textFile.WriteLine("<th>Median (ns)</th>");
textFile.WriteLine("<th>Max (ns)</th>");
textFile.WriteLine("<th>Outliers</th>");
textFile.WriteLine("<th>Std Dev (ns)</th>");
textFile.WriteLine("<th>Std Err (ns)</th>");
textFile.WriteLine("<th>Gen 1</th>");
textFile.WriteLine("<th>Gen 2</th>");
textFile.WriteLine("<th>Bytes Alloc</th>");
textFile.WriteLine("</tr>");
textFile.WriteLine("</thead>");
textFile.WriteLine("<tbody>");
foreach (var report in reports)
{
if (report.ResultStatistics != null)
{
textFile.Write("<tr>");
textFile.Write($"<td>{CreateContainerHtml(report.Benchmark.Target.Method.Name)}</td>");
textFile.Write($"<td>{report.Benchmark.Job.ResolvedId}</td>");
textFile.Write($"<td style=\"text-align: right\">{report.ResultStatistics.Mean:F1}</td>");
textFile.Write($"<td style=\"text-align: right\">{report.ResultStatistics.Median:F1}</td>");
textFile.Write($"<td style=\"text-align: right\">{report.ResultStatistics.Max:F1}</td>");
textFile.Write($"<td style=\"text-align: right\">{report.ResultStatistics.Outliers?.Length ?? 0}</td>");
textFile.Write(
$"<td style= \"text-align: right\">{report.ResultStatistics.StandardDeviation:F3}</td>");
textFile.Write($"<td style=\"text-align: right\">{report.ResultStatistics.StandardError:F3}</td>");
textFile.Write($"<td style=\"text-align: right\">{report.GcStats.Gen1Collections}</td>");
textFile.Write($"<td style=\"text-align: right\">{report.GcStats.Gen2Collections}</td>");
textFile.Write($"<td style=\"text-align: right\">{report.GcStats.BytesAllocatedPerOperation}</td>");
textFile.WriteLine("</tr>");
}
}
textFile.WriteLine("</tbody>");
textFile.Indent(4);
textFile.WriteLine("</table>");
}
public static BenchmarkStatisticsResult simulate(PredictorInfo predictorInfo, BenchmarkInfo benchmarkInfo, SimulationOptions simulationParameters, ApplicationOptions applicationOptions)
{
if (simulationParameters == null)
{
simulationParameters = SimulationOptions.defaultOptions;
}
string folder = "";
ITraceReader traceReader = null;
BenchmarkStatisticsResult currentResult = null;
switch (benchmarkInfo.benchmarkType)
{
case BenchmarkType.Stanford:
folder = applicationOptions.TracePathStanford;
traceReader = new StanfordReader();
break;
case BenchmarkType.SPEC2000:
folder = applicationOptions.TracePathSpec2000;
traceReader = new Spec2000Reader();
break;
case BenchmarkType.CBP2:
folder = applicationOptions.TracePathCBP2;
traceReader = new CBP2Reader();
break;
}
// obtain predictor
IPredictor predictor = predictorInfo.getPredictor();
if (predictor == null)
{
return(new BenchmarkStatisticsResult(benchmarkInfo.benchmarkName + " - Could not be performed: The predictor could not be instantiated on the client."));
}
// open the trace file
if (!traceReader.openTrace(folder, benchmarkInfo.benchmarkName))
{
return(new BenchmarkStatisticsResult(benchmarkInfo.benchmarkName + " - Could not be performed: The trace file could not be opened on the client."));
}
// create new entry
currentResult = new BenchmarkStatisticsResult(benchmarkInfo.benchmarkName);
currentResult.NumberOfCorrectPredictions = 0;
currentResult.NumberOfIncorrectPredictions = 0;
// check if we should skip N branches when counting the performance (to avoid accounting for the predictor warmup)
int branchIndex = 0;
int numberOfBranchesToSkip = simulationParameters.NumberOfBranchesToSkip;
while (true)
{
// get the next branch
IBranch branch = traceReader.getNextBranch();
// null means trace end
if (branch == null)
{
break;
}
BranchInfo currentJumpInfo = branch.getBranchInfo();
if (!simulationParameters.ConditionalOnly || ((branch.getBranchInfo().branchFlags & BranchInfo.BR_CONDITIONAL) > 0))
{
// predict
bool prediction = predictor.predictBranch(branch.getBranchInfo());
if (branchIndex >= numberOfBranchesToSkip)
{
if (branch.taken() == prediction)
{
currentResult.NumberOfCorrectPredictions++;
}
else
{
currentResult.NumberOfIncorrectPredictions++;
}
}
// update the predictor
predictor.update(branch);
branchIndex++;
}
}
traceReader.closeTrace();
currentResult.NumberOfBranches = currentResult.NumberOfCorrectPredictions + currentResult.NumberOfIncorrectPredictions;
currentResult.Accuracy = (double)currentResult.NumberOfCorrectPredictions / currentResult.NumberOfBranches;
// free the predictor instance for future reusability
predictorInfo.freePredictor(predictor);
return(currentResult);
}
