private void WriteBenchmarkIntro(StreamWriter textFile, Summary summary, BenchmarkInfo info) { var benchmark = summary.Benchmarks.First(b => b.Target.Type.Name == info.ClassName); var property = benchmark.Target.Type.GetTypeInfo().DeclaredProperties.FirstOrDefault(p => p.Name == "Description"); var description = (string)property?.GetValue(null) ?? ""; if (info.Parameters != null && info.Parameters.Count > 0) { foreach (var parameter in info.Parameters) { var replaceName = "{" + parameter.Name + "}"; description = description.Replace(replaceName, parameter.Value?.ToString()); } } var link = benchmark.Target.Type.FullName.Replace(nameof(DotNet) + "." + nameof(DependencyInjectionBenchmarks), ""); link = link.Replace('.', '/'); link = CodePath + link + ".cs"; textFile.WriteLine("<div class=\"jumbotron\">"); textFile.WriteLine($"<h1>{info.DisplayName}</h1>"); textFile.WriteLine($"<p>{description} The code can be found <a href=\"{link}\">here</a>.</p>"); textFile.WriteLine("</div>"); }
public void setResult(PredictorInfo predictor, BenchmarkInfo benchmark, BenchmarkStatisticsResult value) { int predictorIndex; int benchmarkIndex; if (!predictorIndexMap.TryGetValue(predictor, out predictorIndex)) { throw new Exception("Invalid predictor provided."); } if (!benchmarkIndexMap.TryGetValue(benchmark, out benchmarkIndex)) { throw new Exception("Invalid simulator provided."); } if ((value != null) && (resultMatrix[predictorIndex, benchmarkIndex] == null)) { valuesEntered++; observableCollections[predictor].addSorted(value); if (newValueReceived != null) { newValueReceived(this, new SimulationResultsDictionaryNewValueReceivedEventArgs(predictorIndex, benchmarkIndex, value, observableCollections[predictor])); } } else { throw new NotSupportedException(); } resultMatrix[predictorIndex, benchmarkIndex] = value; if ((valuesEntered == numberOfPredictors * numberOfBenchmarks) && (filled != null)) { filled(this, EventArgs.Empty); } }
public void addBenchmark(BenchmarkInfo benchmark) { try { benchmarkIndexMap.Add(benchmark, numberOfBenchmarks++); } catch (Exception e) { } }
public IEnumerable <BenchmarkInfo> GetBenchmarksByIsoCountryCode(String isoCode) { IEnumerable <BenchmarkInfo> found; if (this.benchmarkByCountry.Contains(isoCode)) { found = this.benchmarkByCountry[isoCode]; } else { found = new BenchmarkInfo[] { }; } return(found); }
private void Run(BenchmarkInfo benchmarkInfo) { switch (Stage) { case StageEnum.Build: benchmarkInfo.BuildSchema(); break; case StageEnum.TypicalExecution: _documentExecuter.ExecuteAsync(o => { o.Schema = benchmarkInfo.Schema; o.Query = benchmarkInfo.Query; o.Inputs = benchmarkInfo.InputsString?.ToInputs(); }).GetAwaiter().GetResult(); break; case StageEnum.Parse: benchmarkInfo.Parse(); break; case StageEnum.Convert: benchmarkInfo.Convert(); break; case StageEnum.Validate: benchmarkInfo.Validate(); break; case StageEnum.DeserializeVars: benchmarkInfo.DeserializeInputs(); break; case StageEnum.ParseVariables: benchmarkInfo.ParseVariables(); break; case StageEnum.Execute: benchmarkInfo.Execute(); break; case StageEnum.Serialize: benchmarkInfo.Serialize(); break; default: throw new InvalidOperationException(); } }
public BenchmarkStatisticsResult getResult(PredictorInfo predictor, BenchmarkInfo benchmark) { int predictorIndex; int benchmarkIndex; if (!predictorIndexMap.TryGetValue(predictor, out predictorIndex)) { throw new Exception("Invalid predictor provided."); } if (!benchmarkIndexMap.TryGetValue(benchmark, out benchmarkIndex)) { throw new Exception("Invalid simulator provided."); } return(resultMatrix[predictorIndex, benchmarkIndex]); }
public void GlobalSetup() { Func <ISchema> starWarsSchemaBuilder = () => { var services = new ServiceCollection(); services.AddSingleton <StarWarsData>(); services.AddSingleton <StarWarsQuery>(); services.AddSingleton <StarWarsMutation>(); services.AddSingleton <HumanType>(); services.AddSingleton <HumanInputType>(); services.AddSingleton <DroidType>(); services.AddSingleton <CharacterInterface>(); services.AddSingleton <EpisodeEnum>(); services.AddSingleton <ISchema, StarWarsSchema>(); var provider = services.BuildServiceProvider(); var schema = provider.GetRequiredService <ISchema>(); schema.Initialize(); return(schema); }; _bIntrospection = new BenchmarkInfo(Queries.Introspection, null, starWarsSchemaBuilder); _bHero = new BenchmarkInfo(Queries.Hero, null, starWarsSchemaBuilder); Func <ISchema> variableSchemaBuilder = () => { var services = new ServiceCollection(); services.AddSingleton <VariableBenchmark.MyQueryGraphType>(); services.AddSingleton <VariableBenchmark.MyInputObjectGraphType>(); services.AddSingleton <VariableBenchmark.MySubInputObjectGraphType>(); services.AddSingleton <ISchema, VariableBenchmark.MySchema>(); var provider = services.BuildServiceProvider(); var schema = provider.GetRequiredService <ISchema>(); schema.Initialize(); return(schema); }; _bVariable = new BenchmarkInfo(Queries.VariablesVariable, Benchmarks.Variables.VariablesVariable, variableSchemaBuilder); _bLiteral = new BenchmarkInfo(Queries.VariablesLiteral, null, variableSchemaBuilder); }
private void Run(BenchmarkInfo benchmarkInfo) { switch (Stage) { case StageEnum.Build: benchmarkInfo.BuildSchema(); break; case StageEnum.Parse: benchmarkInfo.Parse(); break; case StageEnum.Convert: benchmarkInfo.Convert(); break; case StageEnum.Validate: benchmarkInfo.Validate(); break; case StageEnum.DeserializeVars: benchmarkInfo.DeserializeInputs(); break; case StageEnum.ParseVariables: benchmarkInfo.ParseVariables(); break; case StageEnum.Execute: benchmarkInfo.Execute(); break; case StageEnum.Serialize: benchmarkInfo.Serialize(); break; default: throw new InvalidOperationException(); } }
public SimulationInfo(PredictorInfo predictorInfo, string benchmarkName, BenchmarkType benchmarkType) { this.predictorInfo = predictorInfo; this.benchmarkInfo = new BenchmarkInfo(benchmarkName, benchmarkType); }
public SimulationInfo(PredictorInfo predictorInfo, BenchmarkInfo benchmarkInfo) { this.predictorInfo = predictorInfo; this.benchmarkInfo = benchmarkInfo; }
private void ExportBenchmarkPage(string path, Summary summary, SummaryInfo calculatedHeaders, BenchmarkInfo info) { using (var file = File.Create(Path.Combine(path, info.BenchmarkName.Replace('|', '_') + ".html"))) { using (var textFile = new StreamWriter(file)) { textFile.WriteLine("<html>"); textFile.Indent(1); textFile.WriteLine("<head>"); textFile.Indent(2); textFile.Write(_headString); textFile.Indent(2); textFile.WriteLine("</head>"); textFile.Indent(1); textFile.WriteLine("<body>"); WriteMenu(summary, calculatedHeaders, textFile); textFile.Indent(2); textFile.WriteLine("<div class=\"container-fluid theme-showcase\" role=\"main\">"); WriteBenchmarkIntro(textFile, summary, info); WriteBenchmarkTableData(textFile, summary, info); OutputSummaryDetails(summary, textFile); var environmentSwitches = BuildEnvironmentSwitches(summary); textFile.Write(_scriptString.Replace("{AppendEnvironments}", environmentSwitches)); textFile.Indent(1); textFile.WriteLine("</div></body>"); textFile.WriteLine("</html>"); } } }
private void WriteBenchmarkTableData(StreamWriter textFile, Summary summary, BenchmarkInfo info) { var reports = summary.Reports.Where(r => r.Benchmark.Target.Type.Name == info.ClassName); if (info.Parameters.Count > 0) { reports = reports.Where(r => { foreach (var parameter in info.Parameters) { if (!r.Benchmark.Parameters.Items.Any( p => p.Name == parameter.Name && p.Value == parameter.Value)) { return(false); } } return(true); }); } textFile.Indent(4); textFile.WriteLine("<table class=\"table table-striped table-bordered responsive nowrap\" id=\"benchmarkDataTable\" width=\"100%\" cellspacing=\"0\">"); textFile.WriteLine("<thead>"); textFile.WriteLine("<tr>"); textFile.WriteLine("<th>Container</th>"); textFile.WriteLine("<th>Env</th>"); textFile.WriteLine("<th>Mean (ns)</th>"); textFile.WriteLine("<th>Median (ns)</th>"); textFile.WriteLine("<th>Max (ns)</th>"); textFile.WriteLine("<th>Outliers</th>"); textFile.WriteLine("<th>Std Dev (ns)</th>"); textFile.WriteLine("<th>Std Err (ns)</th>"); textFile.WriteLine("<th>Gen 1</th>"); textFile.WriteLine("<th>Gen 2</th>"); textFile.WriteLine("<th>Bytes Alloc</th>"); textFile.WriteLine("</tr>"); textFile.WriteLine("</thead>"); textFile.WriteLine("<tbody>"); foreach (var report in reports) { if (report.ResultStatistics != null) { textFile.Write("<tr>"); textFile.Write($"<td>{CreateContainerHtml(report.Benchmark.Target.Method.Name)}</td>"); textFile.Write($"<td>{report.Benchmark.Job.ResolvedId}</td>"); textFile.Write($"<td style=\"text-align: right\">{report.ResultStatistics.Mean:F1}</td>"); textFile.Write($"<td style=\"text-align: right\">{report.ResultStatistics.Median:F1}</td>"); textFile.Write($"<td style=\"text-align: right\">{report.ResultStatistics.Max:F1}</td>"); textFile.Write($"<td style=\"text-align: right\">{report.ResultStatistics.Outliers?.Length ?? 0}</td>"); textFile.Write( $"<td style= \"text-align: right\">{report.ResultStatistics.StandardDeviation:F3}</td>"); textFile.Write($"<td style=\"text-align: right\">{report.ResultStatistics.StandardError:F3}</td>"); textFile.Write($"<td style=\"text-align: right\">{report.GcStats.Gen1Collections}</td>"); textFile.Write($"<td style=\"text-align: right\">{report.GcStats.Gen2Collections}</td>"); textFile.Write($"<td style=\"text-align: right\">{report.GcStats.BytesAllocatedPerOperation}</td>"); textFile.WriteLine("</tr>"); } } textFile.WriteLine("</tbody>"); textFile.Indent(4); textFile.WriteLine("</table>"); }
public static BenchmarkStatisticsResult simulate(PredictorInfo predictorInfo, BenchmarkInfo benchmarkInfo, SimulationOptions simulationParameters, ApplicationOptions applicationOptions) { if (simulationParameters == null) { simulationParameters = SimulationOptions.defaultOptions; } string folder = ""; ITraceReader traceReader = null; BenchmarkStatisticsResult currentResult = null; switch (benchmarkInfo.benchmarkType) { case BenchmarkType.Stanford: folder = applicationOptions.TracePathStanford; traceReader = new StanfordReader(); break; case BenchmarkType.SPEC2000: folder = applicationOptions.TracePathSpec2000; traceReader = new Spec2000Reader(); break; case BenchmarkType.CBP2: folder = applicationOptions.TracePathCBP2; traceReader = new CBP2Reader(); break; } // obtain predictor IPredictor predictor = predictorInfo.getPredictor(); if (predictor == null) { return(new BenchmarkStatisticsResult(benchmarkInfo.benchmarkName + " - Could not be performed: The predictor could not be instantiated on the client.")); } // open the trace file if (!traceReader.openTrace(folder, benchmarkInfo.benchmarkName)) { return(new BenchmarkStatisticsResult(benchmarkInfo.benchmarkName + " - Could not be performed: The trace file could not be opened on the client.")); } // create new entry currentResult = new BenchmarkStatisticsResult(benchmarkInfo.benchmarkName); currentResult.NumberOfCorrectPredictions = 0; currentResult.NumberOfIncorrectPredictions = 0; // check if we should skip N branches when counting the performance (to avoid accounting for the predictor warmup) int branchIndex = 0; int numberOfBranchesToSkip = simulationParameters.NumberOfBranchesToSkip; while (true) { // get the next branch IBranch branch = traceReader.getNextBranch(); // null means trace end if (branch == null) { break; } BranchInfo currentJumpInfo = branch.getBranchInfo(); if (!simulationParameters.ConditionalOnly || ((branch.getBranchInfo().branchFlags & BranchInfo.BR_CONDITIONAL) > 0)) { // predict bool prediction = predictor.predictBranch(branch.getBranchInfo()); if (branchIndex >= numberOfBranchesToSkip) { if (branch.taken() == prediction) { currentResult.NumberOfCorrectPredictions++; } else { currentResult.NumberOfIncorrectPredictions++; } } // update the predictor predictor.update(branch); branchIndex++; } } traceReader.closeTrace(); currentResult.NumberOfBranches = currentResult.NumberOfCorrectPredictions + currentResult.NumberOfIncorrectPredictions; currentResult.Accuracy = (double)currentResult.NumberOfCorrectPredictions / currentResult.NumberOfBranches; // free the predictor instance for future reusability predictorInfo.freePredictor(predictor); return(currentResult); }